def test_hash_config_fixes_json(data): with mock.patch('mead.utils.remove_extra_keys') as strip_mock: strip_mock.return_value = data with mock.patch('mead.utils.order_json') as order_mock: order_mock.return_value = data hash_config(data) strip_mock.assert_called_once_with(data) order_mock.assert_called_once_with(data)
def test_hash_config_fixes_json(data): with mock.patch('mead.utils.remove_extra_keys') as strip_mock: strip_mock.return_value = data with mock.patch('mead.utils.order_json') as order_mock: order_mock.return_value = data hash_config(data) strip_mock.assert_called_once_with(data) order_mock.assert_called_once_with(data)
def main(): parser = argparse.ArgumentParser(description="Get the mead hash of a config.") parser.add_argument('config', help='JSON/YML Configuration for an experiment: local file or remote URL', type=convert_path, default="$MEAD_CONFIG") args = parser.parse_args() config = read_config_stream(args.config) print(hash_config(config))
def putmodel(task, eid, cbase, cstore): """ Puts the baseline model files in persistent storage. provide task, id and model base (""tagger-model-tf-"). optionally provide the storage location (/data/model-checkpoints by default) """ ServerManager.get() event_type = EVENT_TYPES['test'] metric = [] try: result = ServerManager.api.experiment_details(task, eid, event_type=event_type, metric=metric) config_obj = read_config_stream(result.config) if config_obj is None: click.echo('can not process the config for experiment {} in {} database'.format(task, eid)) sys.exit(1) result = store_model(checkpoint_base=cbase, config_sha1=hash_config(config_obj), checkpoint_store=cstore, print_fn=click.echo) if result is not None: click.echo(click.style('model stored at {}'.format(result), fg='green')) update_result = ServerManager.api.update_property(task, eid, prop='checkpoint', value=result) if update_result.response_type == 'success': click.echo(click.style(update_result.message, fg='green')) else: click.echo(click.style(update_result.message, fg='red')) else: click.echo(click.style('failed to store model'.format(result), fg='red')) except ApiException as e: click.echo(click.style(json.loads(e.body)['detail'], fg='red'))
def launch(config, settings, logging, hpctl_logging, datasets, embeddings, reporting, unknown, task, num_iters, **kwargs): mead_config = get_config(config, reporting, unknown) hp_settings, mead_settings = get_settings(settings) load_user_modules(mead_config, hp_settings) exp_hash = hash_config(mead_config) hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging) datasets = read_config_file_or_json(datasets) embeddings = read_config_file_or_json(embeddings) if task is None: task = mead_config.get('task', 'classify') _, backend_config = get_ends(hp_settings, unknown) force_remote_backend(backend_config) config_sampler = get_config_sampler(mead_config, None) be = get_backend(backend_config) for _ in range(num_iters): label, config = config_sampler.sample() print(label) send = { 'label': label, 'config': config, 'mead_logs': mead_logs, 'hpctl_logs': hp_logs, 'task_name': task, 'settings': mead_settings, 'experiment_config': mead_config, } be.launch(**send)
def __init__(self, config, results, samplers): super(ConfigSampler, self).__init__() self.config = config self.exp = hash_config(config) self.results = results self.samplers = samplers for sampler in samplers.values(): sampler.values = ConfigSampler._find(config, sampler.name, sampler.adder)
def put_result(self, task, config_obj, events_obj, **kwargs): now = datetime.datetime.utcnow().isoformat() train_events = list(filter(lambda x: x['phase'] == 'Train', events_obj)) valid_events = list(filter(lambda x: x['phase'] == 'Valid', events_obj)) test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj)) checkpoint_base = kwargs.get('checkpoint_base', None) checkpoint_store = kwargs.get('checkpoint_store', None) print_fn = kwargs.get('print_fn', print) hostname = kwargs.get('hostname', socket.gethostname()) username = kwargs.get('username', getpass.getuser()) config_sha1 = hash_config(config_obj) label = get_experiment_label(config_obj, task, **kwargs) post = { "config": config_obj, "train_events": train_events, "valid_events": valid_events, "test_events": test_events, "username": username, "hostname": hostname, "date": now, "label": label, "sha1": config_sha1, "version": __version__ } if checkpoint_base: model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store) if model_loc is not None: post.update({ "checkpoint": "{}:{}".format(hostname, os.path.abspath(model_loc)) }) else: print_fn("model could not be stored, see previous errors") if task in self.db.collection_names(): print_fn( "updating results for existing task [{}] in host [{}]".format( task, self.dbhost)) else: print_fn("creating new task [{}] in host [{}]".format( task, self.dbhost)) coll = self.db[task] result = coll.insert_one(post) print_fn( "results updated, the new results are stored with the record id: {}" .format(result.inserted_id)) return result.inserted_id
def put_result(self, task, config_obj, events_obj, **kwargs): session = self.Session() print_fn = kwargs.get('print_fn', print) now = datetime.datetime.utcnow().isoformat() hostname = kwargs.get('hostname', socket.gethostname()) username = kwargs.get('username', getpass.getuser()) config_sha1 = hash_config(config_obj) label = get_experiment_label(config_obj, task, **kwargs) checkpoint_base = kwargs.get('checkpoint_base', None) checkpoint_store = kwargs.get('checkpoint_store', None) checkpoint = None if checkpoint_base: model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store) if model_loc is not None: checkpoint = "{}:{}".format(hostname, os.path.abspath(model_loc)) else: print_fn("model could not be stored, see previous errors") event_objs = [] for event in events_obj: tick = event['tick'] date = event.get('date') phase = event['phase'] event_obj = Event(tick=tick, date=date, phase=phase, metrics=[]) for key in event.keys(): if key not in [ 'tick_type', 'tick', 'event_type', 'id', 'date', 'phase' ]: metric = Metric(label=key, value=event[key]) event_obj.metrics += [metric] event_objs += [event_obj] experiment = Experiment(label=label, checkpoint=checkpoint, sha1=config_sha1, task=task, dataset=config_obj['dataset'], config=json.dumps(config_obj), hostname=hostname, username=username, date=now, version=__version__, status='CREATED', last_modified=now) experiment.events = event_objs session.add(experiment) session.commit() return experiment
def test_get_results_by_prop(setup): """ get_results_by_prop first finds some experiments by a (prop, value) pair and then groups them by a reduction_dim. typically we use prop='dataset' and reduction_dim='sha1', but the API supports more general calls :param setup: :return: """ configs = ['{"c1":"c1"}', '{"c2":"c2"}'] datasets = ['d1', 'd1', 'd2'] labels = ['l1', 'l2', 'l2'] metrics = ['f1', 'acc', 'random'] test_events = [ Result(metric=metric, value=0.5, tick_type='EPOCH', tick=0, phase='Test') for metric in metrics ] experiments = [] exp_detail = namedtuple('exp_detail', ['eid', 'sha1', 'dataset', 'label']) for config in configs: for dataset in datasets: for label in labels: result = _put_one_exp( Experiment(config=config, dataset=dataset, label=label, train_events=[], valid_events=[], test_events=test_events)) experiments.append( exp_detail(eid=result, sha1=hash_config(json.loads(config)), dataset=dataset, label=label)) # find by a property and group by different reduction dims for prop, values in {'dataset': datasets, 'label': labels}.items(): for value in values: for reduction_dim in ['sha1', 'eid', 'label']: if prop == 'dataset': assert _test_reduction_dim_dataset( dataset_value=value, reduction_dim=reduction_dim, experiments=experiments) else: assert _test_reduction_dim_label( label_value=value, reduction_dim=reduction_dim, experiments=experiments)
def put_result(self, task, config_obj, events_obj, **kwargs): session = self.Session() print_fn = kwargs.get('print_fn', print) now = datetime.datetime.utcnow().isoformat() hostname = kwargs.get('hostname', socket.gethostname()) username = kwargs.get('username', getpass.getuser()) config_sha1 = hash_config(config_obj) label = get_experiment_label(config_obj, task, **kwargs) checkpoint_base = kwargs.get('checkpoint_base', None) checkpoint_store = kwargs.get('checkpoint_store', None) checkpoint = None if checkpoint_base: model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store) if model_loc is not None: checkpoint = "{}:{}".format(hostname, os.path.abspath(model_loc)) else: print_fn("model could not be stored, see previous errors") event_objs = [] for event in events_obj: tick = event['tick'] date = event.get('date') phase = event['phase'] event_obj = Event(tick=tick, date=date, phase=phase, metrics=[]) for key in event.keys(): if key not in ['tick_type', 'tick', 'event_type', 'id', 'date', 'phase']: metric = Metric(label=key, value=event[key]) event_obj.metrics += [metric] event_objs += [event_obj] experiment = Experiment( label=label, checkpoint=checkpoint, sha1=config_sha1, task=task, dataset=config_obj['dataset'], config=json.dumps(config_obj), hostname=hostname, username=username, date=now, version=__version__, status='CREATED', last_modified=now ) experiment.events = event_objs session.add(experiment) session.commit() return experiment
def done(self): """Write the log to the xpctl database""" try: result = self.api.put_result( self.task, to_experiment( self.task, self.exp_config, self.log, username=self.username, label=self.label, hostname=self.hostname, )) if result.response_type == 'failure': raise RuntimeError(result.message) else: print('result stored with experiment id', result.message) if self.save_model: eid = result.message backend = self.exp_config.get('backend', 'default') backends = { 'default': 'tf', 'tensorflow': 'tf', 'pytorch': 'pyt' } self.checkpoint_base = self._search_checkpoint_base( self.task, backends[backend], self.exp_config.get('basedir')) if self.checkpoint_base is None: raise RuntimeError('No checkpoint files found') result = store_model(checkpoint_base=self.checkpoint_base, config_sha1=hash_config(self.exp_config), checkpoint_store=self.checkpoint_store) if result is not None: print('model stored at {}'.format(result)) update_result = self.api.update_property(self.task, eid, prop='checkpoint', value=result) print(update_result.message) else: raise RuntimeError('failed to store model at {}'.format( self.checkpoint_store)) except ApiException as e: raise RuntimeError(json.loads(e.body)['detail'])
def sample(self): """Replace values with the sampled ones. :returns: tuple, (hpctl.utils.Label dict): [0]: The label [3]: The config. """ s = deepcopy(self.config) for sampler in self.samplers.values(): sampled = sampler.sample() for k, v in sampled.items(): tmp = s for x in k[:-1]: tmp = tmp[x] tmp[k[-1]] = v hashed = hash_config(s) label = Label(self.exp, hashed, random_name()) return label, s
def _put_result(self, task, config_obj, events_obj, **kwargs): session = self.Session() now = safe_get(kwargs, 'date', datetime.datetime.utcnow().isoformat()) hostname = safe_get(kwargs, 'hostname', socket.gethostname()) username = safe_get(kwargs, 'username', getpass.getuser()) config_sha1 = safe_get(kwargs, 'sha1', hash_config(config_obj)) label = safe_get(kwargs, 'label', get_experiment_label(config_obj, task, **kwargs)) checkpoint = kwargs.get('checkpoint') version = safe_get(kwargs, 'version', __version__) dataset = safe_get(kwargs, 'dataset', config_obj.get('dataset')) date = safe_get(kwargs, 'exp_date', now) events = [] for event in events_obj: tick = event['tick'] phase = event['phase'] event_obj = SqlEvent(phase=phase, tick_type=event['tick_type'], tick=tick, results=[]) for key in event.keys(): if key not in ['tick_type', 'tick', 'event_type', 'id', 'date', 'phase']: metric = SqlResult(metric=key, value=event[key]) event_obj.results += [metric] events += [event_obj] experiment = SqlExperiment( label=label, checkpoint=checkpoint, sha1=config_sha1, task=task, dataset=dataset, config=json.dumps(config_obj), hostname=hostname, username=username, date=date, version=version, status='CREATED', last_modified=now, events=events ) try: session.add(experiment) session.commit() return BackendSuccess(message=experiment.eid) except sql.exc.SQLAlchemyError as e: return BackendError(message=str(e))
def put_result(self, task, config_obj, events_obj, **kwargs): now = datetime.datetime.utcnow().isoformat() train_events = list(filter(lambda x: x['phase'] == 'Train', events_obj)) valid_events = list(filter(lambda x: x['phase'] == 'Valid', events_obj)) test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj)) checkpoint_base = kwargs.get('checkpoint_base', None) checkpoint_store = kwargs.get('checkpoint_store', None) print_fn = kwargs.get('print_fn', print) hostname = kwargs.get('hostname', socket.gethostname()) username = kwargs.get('username', getpass.getuser()) config_sha1 = hash_config(config_obj) label = get_experiment_label(config_obj, task, **kwargs) post = { "config": config_obj, "train_events": train_events, "valid_events": valid_events, "test_events": test_events, "username": username, "hostname": hostname, "date": now, "label": label, "sha1": config_sha1, "version": __version__ } if checkpoint_base: model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store) if model_loc is not None: post.update({"checkpoint": "{}:{}".format(hostname, os.path.abspath(model_loc))}) else: print_fn("model could not be stored, see previous errors") if task in self.db.collection_names(): print_fn("updating results for existing task [{}] in host [{}]".format(task, self.dbhost)) else: print_fn("creating new task [{}] in host [{}]".format(task, self.dbhost)) coll = self.db[task] result = coll.insert_one(post) print_fn("results updated, the new results are stored with the record id: {}".format(result.inserted_id)) return result.inserted_id
def putresult(task, config, log, dataset, user, label, cbase, cstore): """ Puts the results in a database. provide task name, config file, the reporting log file, and the dataset index file used in the experiment. Optionally can put the model files in a persistent storage. """ logf = log.format(task) if not os.path.exists(logf): click.echo(click.style("the log file at {} doesn't exist, provide a valid location".format(logf), fg='red')) return if not os.path.exists(config): click.echo(click.style("the config file at {} doesn't exist, provide a valid location".format(config), fg='red')) return if not os.path.exists(dataset): click.echo(click.style("the dataset file at {} doesn't exist, provide a valid location".format(dataset), fg='red')) return config_obj = read_config_file(config) datasets_set = index_by_label(read_config_file(dataset)) dataset_key = config_obj['dataset'] dataset_key = get_dataset_from_key(dataset_key, datasets_set) config_obj['dataset'] = dataset_key['label'] ServerManager.get() result = ServerManager.api.put_result(task, to_swagger_experiment(task, config_obj, log, username=user, label=label)) if result.response_type == 'success': eid = result.message click.echo(click.style('results stored with experiment: {}'.format(result.message), fg='green')) if cbase is None: return result = store_model(checkpoint_base=cbase, config_sha1=hash_config(read_config_file(config)), checkpoint_store=cstore, print_fn=click.echo) if result is not None: click.echo(click.style('model stored at {}'.format(result), fg='green')) update_result = ServerManager.api.update_property(task, eid, prop='checkpoint', value=result) if update_result.response_type == 'success': click.echo(click.style(update_result.message, fg='green')) else: click.echo(click.style(update_result.message, fg='red')) else: click.echo(click.style('failed to store model'.format(result), fg='red')) else: click.echo(click.style(result.message, fg='red'))
def _put_result(self, task, config_obj, events_obj, **kwargs): now = safe_get(kwargs, 'date', datetime.datetime.utcnow().isoformat()) hostname = safe_get(kwargs, 'hostname', socket.gethostname()) username = safe_get(kwargs, 'username', getpass.getuser()) config_sha1 = safe_get(kwargs, 'sha1', hash_config(config_obj)) label = safe_get(kwargs, 'label', get_experiment_label(config_obj, task, **kwargs)) checkpoint = kwargs.get('checkpoint') version = safe_get(kwargs, 'version', __version__) dataset = safe_get(kwargs, 'dataset', config_obj.get('dataset')) date = safe_get(kwargs, 'exp_date', now) train_events = list(filter(lambda x: x['phase'] == 'Train', events_obj)) valid_events = list(filter(lambda x: x['phase'] == 'Valid', events_obj)) test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj)) post = { "dataset": dataset, "config": config_obj, "train_events": train_events, "valid_events": valid_events, "test_events": test_events, "username": username, "hostname": hostname, "date": date, "label": label, "sha1": config_sha1, "version": version, "checkpoint": checkpoint } if 'eid' in kwargs: post.update({'_id': ObjectId(kwargs['eid'])}) try: coll = self.db[task] result = coll.insert_one(post) return BackendSuccess(message=str(result.inserted_id)) except pymongo.errors.PyMongoError as e: return BackendError(message='experiment could not be inserted: {}'.format(e.message))
def launch( config, settings, logging, hpctl_logging, datasets, embeddings, reporting, unknown, task, num_iters, **kwargs ): mead_config = get_config(config, reporting, unknown) hp_settings, mead_settings = get_settings(settings) load_user_modules(mead_config, hp_settings) exp_hash = hash_config(mead_config) hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging) datasets = read_config_file_or_json(datasets) embeddings = read_config_file_or_json(embeddings) if task is None: task = mead_config.get('task', 'classify') _, backend_config = get_ends(hp_settings, unknown) force_remote_backend(backend_config) config_sampler = get_config_sampler(mead_config, None) be = get_backend(backend_config) for _ in range(num_iters): label, config = config_sampler.sample() print(label) send = { 'label': label, 'config': config, 'mead_logs': mead_logs, 'hpctl_logs': hp_logs, 'task_name': task, 'settings': mead_settings, 'experiment_config': mead_config, } be.launch(**send)
def test_experiment_details(setup): date = datetime.datetime.utcnow().isoformat() label = 'test_label' dataset = 'test_dataset' config = '{"test":"test"}' username = '******' hostname = 'host' exp = Experiment(dataset=dataset, label=label, exp_date=date, config=config, username=username, hostname=hostname, train_events=[], valid_events=[], test_events=[ Result(metric='t', tick_type='t', phase='Test', tick=0, value=0.1) ]) eid = _put_one_exp(exp) try: result = API.list_experiments_by_prop(TASK, eid=eid) except ApiException: print(eid) return False exp = result[0] assert exp.dataset == 'test_dataset' assert exp.label == label assert (exp.exp_date == date) or (exp.exp_date[:-1] == date ) # sql inserts Z at the end assert exp.sha1 == hash_config(json.loads('{"test": "test"}')) assert exp.username == username assert exp.hostname == hostname
def search( config, settings, logging, hpctl_logging, datasets, embeddings, reporting, unknown, task, num_iters, **kwargs ): """Search for optimal hyperparameters.""" mead_config = get_config(config, reporting, unknown) hp_settings, mead_settings = get_settings(settings) load_user_modules(mead_config, hp_settings) exp_hash = hash_config(mead_config) hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging) datasets = read_config_file_or_json(datasets) embeddings = read_config_file_or_json(embeddings) if task is None: task = mead_config.get('task', 'classify') frontend_config, backend_config = get_ends(hp_settings, unknown) # Figure out xpctl xpctl_config = None auto_xpctl = 'xpctl' in mead_config.get('reporting', []) if not auto_xpctl: # If the jobs aren't setup to use xpctl automatically create your own xpctl_config = get_xpctl_settings(mead_settings) if xpctl_config is not None: xpctl_extra = parse_extra_args(['xpctl'], unknown) xpctl_config['label'] = xpctl_extra.get('xpctl', {}).get('label') results_config = {} # Set frontend defaults frontend_config['experiment_hash'] = exp_hash default = mead_config['train'].get('early_stopping_metric', 'avg_loss') frontend_config.setdefault('train', 'avg_loss') frontend_config.setdefault('dev', default) frontend_config.setdefault('test', default) # Negotiate remote status if backend_config['type'] != 'remote': set_root(hp_settings) _remote_monkey_patch(backend_config, hp_logs, results_config, xpctl_config) xpctl = get_xpctl(xpctl_config) results = get_results(results_config) results.add_experiment(mead_config) backend = get_backend(backend_config) config_sampler = get_config_sampler(mead_config, results) logs = get_log_server(hp_logs) frontend = get_frontend(frontend_config, results, xpctl) labels = run(num_iters, results, backend, frontend, config_sampler, logs, mead_logs, hp_logs, mead_settings, datasets, embeddings, task) logs.stop() frontend.finalize() results.save() if auto_xpctl: for label in labels: results.set_xpctl(label, True) return labels, results
def add_experiment(self, exp_config): exp_hash = hash_config(exp_config) self.exp_to_config[exp_hash] = exp_config
def test_list_experiments_by_prop(setup): """ list_experiments_by_prop finds some experiments by a (prop, value) pair. also allows some filtering: :param setup: :return: """ configs = ['{"c1":"c1"}', '{"c2":"c2"}'] datasets = ['d1', 'd2'] labels = ['l1', 'l2'] users = ['u1', 'u2', 'u3'] metrics = ['f1', 'acc', 'random'] test_events = [ Result(metric=metric, value=0.5, tick_type='EPOCH', tick=0, phase='Test') for metric in metrics ] experiments = [] exp_detail = namedtuple('exp_detail', ['eid', 'sha1', 'dataset', 'label', 'username']) for config in configs: for dataset in datasets: for label in labels: for username in users: result = _put_one_exp( Experiment(config=config, dataset=dataset, label=label, username=username, train_events=[], valid_events=[], test_events=test_events)) experiments.append( exp_detail(eid=result, sha1=hash_config(json.loads(config)), dataset=dataset, label=label, username=username)) # find by a property and group by different reduction dims for prop, values in { 'dataset': datasets, 'label': labels, 'username': users }.items(): for value in values: assert _test_list_experiments_by_prop(prop=prop, value=value, experiments=experiments) # test the `users` filter work prop = 'dataset' value = 'd1' users = [['u1', 'u2'], ['u1']] for user in users: results = API.list_experiments_by_prop(TASK, dataset=value, user=user) result_eids = [x.eid for x in results] expected_eids = [ x.eid for x in experiments if x.dataset == 'd1' and x.username in user ] assert set(result_eids) == set(expected_eids) # test sort works config = configs[0] metrics = ['f1', 'acc'] exp_value = namedtuple('exp_value', ['eid', 'value']) experiments = [] dataset = _generate_random_string() # generate a unique dataset for value in [0.5, 0.7, 0.8]: test_events = [ Result(metric=metric, value=value, tick_type='EPOCH', tick=0, phase='Test') for metric in metrics ] result = _put_one_exp( Experiment(config=config, train_events=[], valid_events=[], test_events=test_events, dataset=dataset)) experiments.append(exp_value(eid=result, value=value)) results = API.list_experiments_by_prop( TASK, dataset=dataset, sort='f1') # get results only from that # unique dataset, sort by f1 _max = experiments[-1] assert results[0].eid == _max.eid
def search(config, settings, logging, hpctl_logging, datasets, embeddings, reporting, unknown, task, num_iters, **kwargs): """Search for optimal hyperparameters.""" mead_config = get_config(config, reporting, unknown) hp_settings, mead_settings = get_settings(settings) load_user_modules(mead_config, hp_settings) exp_hash = hash_config(mead_config) hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging) datasets = read_config_file_or_json(datasets) embeddings = read_config_file_or_json(embeddings) if task is None: task = mead_config.get('task', 'classify') frontend_config, backend_config = get_ends(hp_settings, unknown) # Figure out xpctl xpctl_config = None auto_xpctl = 'xpctl' in mead_config.get('reporting', []) if not auto_xpctl: # If the jobs aren't setup to use xpctl automatically create your own xpctl_config = get_xpctl_settings(mead_settings) if xpctl_config is not None: xpctl_extra = parse_extra_args(['xpctl'], unknown) xpctl_config['label'] = xpctl_extra.get('xpctl', {}).get('label') results_config = {} # Set frontend defaults frontend_config['experiment_hash'] = exp_hash default = mead_config['train'].get('early_stopping_metric', 'avg_loss') frontend_config.setdefault('train', 'avg_loss') frontend_config.setdefault('dev', default) frontend_config.setdefault('test', default) # Negotiate remote status if backend_config['type'] != 'remote': set_root(hp_settings) _remote_monkey_patch(backend_config, hp_logs, results_config, xpctl_config) xpctl = get_xpctl(xpctl_config) results = get_results(results_config) results.add_experiment(mead_config) backend = get_backend(backend_config) config_sampler = get_config_sampler(mead_config, results) logs = get_log_server(hp_logs) frontend = get_frontend(frontend_config, results, xpctl) labels = run(num_iters, results, backend, frontend, config_sampler, logs, mead_logs, hp_logs, mead_settings, datasets, embeddings, task) logs.stop() frontend.finalize() results.save() if auto_xpctl: for label in labels: results.set_xpctl(label, True) return labels, results