def test_hash_config_fixes_json(data):
with mock.patch('mead.utils.remove_extra_keys') as strip_mock:
strip_mock.return_value = data
with mock.patch('mead.utils.order_json') as order_mock:
order_mock.return_value = data
hash_config(data)
strip_mock.assert_called_once_with(data)
order_mock.assert_called_once_with(data)
def test_hash_config_fixes_json(data):
with mock.patch('mead.utils.remove_extra_keys') as strip_mock:
strip_mock.return_value = data
with mock.patch('mead.utils.order_json') as order_mock:
order_mock.return_value = data
hash_config(data)
strip_mock.assert_called_once_with(data)
order_mock.assert_called_once_with(data)
def main():
parser = argparse.ArgumentParser(description="Get the mead hash of a config.")
parser.add_argument('config', help='JSON/YML Configuration for an experiment: local file or remote URL', type=convert_path, default="$MEAD_CONFIG")
args = parser.parse_args()
config = read_config_stream(args.config)
print(hash_config(config))
def putmodel(task, eid, cbase, cstore):
"""
Puts the baseline model files in persistent storage. provide task, id and model base (""tagger-model-tf-").
optionally provide the storage location (/data/model-checkpoints by default)
"""
ServerManager.get()
event_type = EVENT_TYPES['test']
metric = []
try:
result = ServerManager.api.experiment_details(task, eid, event_type=event_type, metric=metric)
config_obj = read_config_stream(result.config)
if config_obj is None:
click.echo('can not process the config for experiment {} in {} database'.format(task, eid))
sys.exit(1)
result = store_model(checkpoint_base=cbase, config_sha1=hash_config(config_obj),
checkpoint_store=cstore, print_fn=click.echo)
if result is not None:
click.echo(click.style('model stored at {}'.format(result), fg='green'))
update_result = ServerManager.api.update_property(task, eid, prop='checkpoint', value=result)
if update_result.response_type == 'success':
click.echo(click.style(update_result.message, fg='green'))
else:
click.echo(click.style(update_result.message, fg='red'))
else:
click.echo(click.style('failed to store model'.format(result), fg='red'))
except ApiException as e:
click.echo(click.style(json.loads(e.body)['detail'], fg='red'))
def launch(config, settings, logging, hpctl_logging, datasets, embeddings,
reporting, unknown, task, num_iters, **kwargs):
mead_config = get_config(config, reporting, unknown)
hp_settings, mead_settings = get_settings(settings)
load_user_modules(mead_config, hp_settings)
exp_hash = hash_config(mead_config)
hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging)
datasets = read_config_file_or_json(datasets)
embeddings = read_config_file_or_json(embeddings)
if task is None:
task = mead_config.get('task', 'classify')
_, backend_config = get_ends(hp_settings, unknown)
force_remote_backend(backend_config)
config_sampler = get_config_sampler(mead_config, None)
be = get_backend(backend_config)
for _ in range(num_iters):
label, config = config_sampler.sample()
print(label)
send = {
'label': label,
'config': config,
'mead_logs': mead_logs,
'hpctl_logs': hp_logs,
'task_name': task,
'settings': mead_settings,
'experiment_config': mead_config,
}
be.launch(**send)
def __init__(self, config, results, samplers):
super(ConfigSampler, self).__init__()
self.config = config
self.exp = hash_config(config)
self.results = results
self.samplers = samplers
for sampler in samplers.values():
sampler.values = ConfigSampler._find(config, sampler.name, sampler.adder)
def put_result(self, task, config_obj, events_obj, **kwargs):
now = datetime.datetime.utcnow().isoformat()
train_events = list(filter(lambda x: x['phase'] == 'Train',
events_obj))
valid_events = list(filter(lambda x: x['phase'] == 'Valid',
events_obj))
test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj))
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
print_fn = kwargs.get('print_fn', print)
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hash_config(config_obj)
label = get_experiment_label(config_obj, task, **kwargs)
post = {
"config": config_obj,
"train_events": train_events,
"valid_events": valid_events,
"test_events": test_events,
"username": username,
"hostname": hostname,
"date": now,
"label": label,
"sha1": config_sha1,
"version": __version__
}
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1,
checkpoint_store)
if model_loc is not None:
post.update({
"checkpoint":
"{}:{}".format(hostname, os.path.abspath(model_loc))
})
else:
print_fn("model could not be stored, see previous errors")
if task in self.db.collection_names():
print_fn(
"updating results for existing task [{}] in host [{}]".format(
task, self.dbhost))
else:
print_fn("creating new task [{}] in host [{}]".format(
task, self.dbhost))
coll = self.db[task]
result = coll.insert_one(post)
print_fn(
"results updated, the new results are stored with the record id: {}"
.format(result.inserted_id))
return result.inserted_id
def put_result(self, task, config_obj, events_obj, **kwargs):
session = self.Session()
print_fn = kwargs.get('print_fn', print)
now = datetime.datetime.utcnow().isoformat()
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hash_config(config_obj)
label = get_experiment_label(config_obj, task, **kwargs)
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
checkpoint = None
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1,
checkpoint_store)
if model_loc is not None:
checkpoint = "{}:{}".format(hostname,
os.path.abspath(model_loc))
else:
print_fn("model could not be stored, see previous errors")
event_objs = []
for event in events_obj:
tick = event['tick']
date = event.get('date')
phase = event['phase']
event_obj = Event(tick=tick, date=date, phase=phase, metrics=[])
for key in event.keys():
if key not in [
'tick_type', 'tick', 'event_type', 'id', 'date',
'phase'
]:
metric = Metric(label=key, value=event[key])
event_obj.metrics += [metric]
event_objs += [event_obj]
experiment = Experiment(label=label,
checkpoint=checkpoint,
sha1=config_sha1,
task=task,
dataset=config_obj['dataset'],
config=json.dumps(config_obj),
hostname=hostname,
username=username,
date=now,
version=__version__,
status='CREATED',
last_modified=now)
experiment.events = event_objs
session.add(experiment)
session.commit()
return experiment
def test_get_results_by_prop(setup):
"""
get_results_by_prop first finds some experiments by a (prop, value) pair and then groups them by a reduction_dim.
typically we use prop='dataset' and reduction_dim='sha1', but the API supports more general calls
:param setup:
:return:
"""
configs = ['{"c1":"c1"}', '{"c2":"c2"}']
datasets = ['d1', 'd1', 'd2']
labels = ['l1', 'l2', 'l2']
metrics = ['f1', 'acc', 'random']
test_events = [
Result(metric=metric,
value=0.5,
tick_type='EPOCH',
tick=0,
phase='Test') for metric in metrics
]
experiments = []
exp_detail = namedtuple('exp_detail', ['eid', 'sha1', 'dataset', 'label'])
for config in configs:
for dataset in datasets:
for label in labels:
result = _put_one_exp(
Experiment(config=config,
dataset=dataset,
label=label,
train_events=[],
valid_events=[],
test_events=test_events))
experiments.append(
exp_detail(eid=result,
sha1=hash_config(json.loads(config)),
dataset=dataset,
label=label))
# find by a property and group by different reduction dims
for prop, values in {'dataset': datasets, 'label': labels}.items():
for value in values:
for reduction_dim in ['sha1', 'eid', 'label']:
if prop == 'dataset':
assert _test_reduction_dim_dataset(
dataset_value=value,
reduction_dim=reduction_dim,
experiments=experiments)
else:
assert _test_reduction_dim_label(
label_value=value,
reduction_dim=reduction_dim,
experiments=experiments)
def put_result(self, task, config_obj, events_obj, **kwargs):
session = self.Session()
print_fn = kwargs.get('print_fn', print)
now = datetime.datetime.utcnow().isoformat()
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hash_config(config_obj)
label = get_experiment_label(config_obj, task, **kwargs)
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
checkpoint = None
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store)
if model_loc is not None:
checkpoint = "{}:{}".format(hostname, os.path.abspath(model_loc))
else:
print_fn("model could not be stored, see previous errors")
event_objs = []
for event in events_obj:
tick = event['tick']
date = event.get('date')
phase = event['phase']
event_obj = Event(tick=tick, date=date, phase=phase, metrics=[])
for key in event.keys():
if key not in ['tick_type', 'tick', 'event_type', 'id', 'date', 'phase']:
metric = Metric(label=key, value=event[key])
event_obj.metrics += [metric]
event_objs += [event_obj]
experiment = Experiment(
label=label,
checkpoint=checkpoint,
sha1=config_sha1,
task=task,
dataset=config_obj['dataset'],
config=json.dumps(config_obj),
hostname=hostname,
username=username,
date=now,
version=__version__,
status='CREATED',
last_modified=now
)
experiment.events = event_objs
session.add(experiment)
session.commit()
return experiment
def done(self):
"""Write the log to the xpctl database"""
try:
result = self.api.put_result(
self.task,
to_experiment(
self.task,
self.exp_config,
self.log,
username=self.username,
label=self.label,
hostname=self.hostname,
))
if result.response_type == 'failure':
raise RuntimeError(result.message)
else:
print('result stored with experiment id', result.message)
if self.save_model:
eid = result.message
backend = self.exp_config.get('backend', 'default')
backends = {
'default': 'tf',
'tensorflow': 'tf',
'pytorch': 'pyt'
}
self.checkpoint_base = self._search_checkpoint_base(
self.task, backends[backend],
self.exp_config.get('basedir'))
if self.checkpoint_base is None:
raise RuntimeError('No checkpoint files found')
result = store_model(checkpoint_base=self.checkpoint_base,
config_sha1=hash_config(self.exp_config),
checkpoint_store=self.checkpoint_store)
if result is not None:
print('model stored at {}'.format(result))
update_result = self.api.update_property(self.task,
eid,
prop='checkpoint',
value=result)
print(update_result.message)
else:
raise RuntimeError('failed to store model at {}'.format(
self.checkpoint_store))
except ApiException as e:
raise RuntimeError(json.loads(e.body)['detail'])
def sample(self):
"""Replace values with the sampled ones.
:returns: tuple, (hpctl.utils.Label dict):
[0]: The label
[3]: The config.
"""
s = deepcopy(self.config)
for sampler in self.samplers.values():
sampled = sampler.sample()
for k, v in sampled.items():
tmp = s
for x in k[:-1]:
tmp = tmp[x]
tmp[k[-1]] = v
hashed = hash_config(s)
label = Label(self.exp, hashed, random_name())
return label, s
def _put_result(self, task, config_obj, events_obj, **kwargs):
session = self.Session()
now = safe_get(kwargs, 'date', datetime.datetime.utcnow().isoformat())
hostname = safe_get(kwargs, 'hostname', socket.gethostname())
username = safe_get(kwargs, 'username', getpass.getuser())
config_sha1 = safe_get(kwargs, 'sha1', hash_config(config_obj))
label = safe_get(kwargs, 'label', get_experiment_label(config_obj, task, **kwargs))
checkpoint = kwargs.get('checkpoint')
version = safe_get(kwargs, 'version', __version__)
dataset = safe_get(kwargs, 'dataset', config_obj.get('dataset'))
date = safe_get(kwargs, 'exp_date', now)
events = []
for event in events_obj:
tick = event['tick']
phase = event['phase']
event_obj = SqlEvent(phase=phase, tick_type=event['tick_type'], tick=tick, results=[])
for key in event.keys():
if key not in ['tick_type', 'tick', 'event_type', 'id', 'date', 'phase']:
metric = SqlResult(metric=key, value=event[key])
event_obj.results += [metric]
events += [event_obj]
experiment = SqlExperiment(
label=label,
checkpoint=checkpoint,
sha1=config_sha1,
task=task,
dataset=dataset,
config=json.dumps(config_obj),
hostname=hostname,
username=username,
date=date,
version=version,
status='CREATED',
last_modified=now,
events=events
)
try:
session.add(experiment)
session.commit()
return BackendSuccess(message=experiment.eid)
except sql.exc.SQLAlchemyError as e:
return BackendError(message=str(e))
def put_result(self, task, config_obj, events_obj, **kwargs):
now = datetime.datetime.utcnow().isoformat()
train_events = list(filter(lambda x: x['phase'] == 'Train', events_obj))
valid_events = list(filter(lambda x: x['phase'] == 'Valid', events_obj))
test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj))
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
print_fn = kwargs.get('print_fn', print)
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hash_config(config_obj)
label = get_experiment_label(config_obj, task, **kwargs)
post = {
"config": config_obj,
"train_events": train_events,
"valid_events": valid_events,
"test_events": test_events,
"username": username,
"hostname": hostname,
"date": now,
"label": label,
"sha1": config_sha1,
"version": __version__
}
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store)
if model_loc is not None:
post.update({"checkpoint": "{}:{}".format(hostname, os.path.abspath(model_loc))})
else:
print_fn("model could not be stored, see previous errors")
if task in self.db.collection_names():
print_fn("updating results for existing task [{}] in host [{}]".format(task, self.dbhost))
else:
print_fn("creating new task [{}] in host [{}]".format(task, self.dbhost))
coll = self.db[task]
result = coll.insert_one(post)
print_fn("results updated, the new results are stored with the record id: {}".format(result.inserted_id))
return result.inserted_id
def putresult(task, config, log, dataset, user, label, cbase, cstore):
"""
Puts the results in a database. provide task name, config file, the reporting log file, and the dataset index file
used in the experiment. Optionally can put the model files in a persistent storage.
"""
logf = log.format(task)
if not os.path.exists(logf):
click.echo(click.style("the log file at {} doesn't exist, provide a valid location".format(logf), fg='red'))
return
if not os.path.exists(config):
click.echo(click.style("the config file at {} doesn't exist, provide a valid location".format(config), fg='red'))
return
if not os.path.exists(dataset):
click.echo(click.style("the dataset file at {} doesn't exist, provide a valid location".format(dataset), fg='red'))
return
config_obj = read_config_file(config)
datasets_set = index_by_label(read_config_file(dataset))
dataset_key = config_obj['dataset']
dataset_key = get_dataset_from_key(dataset_key, datasets_set)
config_obj['dataset'] = dataset_key['label']
ServerManager.get()
result = ServerManager.api.put_result(task, to_swagger_experiment(task, config_obj, log, username=user, label=label))
if result.response_type == 'success':
eid = result.message
click.echo(click.style('results stored with experiment: {}'.format(result.message), fg='green'))
if cbase is None:
return
result = store_model(checkpoint_base=cbase, config_sha1=hash_config(read_config_file(config)),
checkpoint_store=cstore, print_fn=click.echo)
if result is not None:
click.echo(click.style('model stored at {}'.format(result), fg='green'))
update_result = ServerManager.api.update_property(task, eid, prop='checkpoint', value=result)
if update_result.response_type == 'success':
click.echo(click.style(update_result.message, fg='green'))
else:
click.echo(click.style(update_result.message, fg='red'))
else:
click.echo(click.style('failed to store model'.format(result), fg='red'))
else:
click.echo(click.style(result.message, fg='red'))
def _put_result(self, task, config_obj, events_obj, **kwargs):
now = safe_get(kwargs, 'date', datetime.datetime.utcnow().isoformat())
hostname = safe_get(kwargs, 'hostname', socket.gethostname())
username = safe_get(kwargs, 'username', getpass.getuser())
config_sha1 = safe_get(kwargs, 'sha1', hash_config(config_obj))
label = safe_get(kwargs, 'label', get_experiment_label(config_obj, task, **kwargs))
checkpoint = kwargs.get('checkpoint')
version = safe_get(kwargs, 'version', __version__)
dataset = safe_get(kwargs, 'dataset', config_obj.get('dataset'))
date = safe_get(kwargs, 'exp_date', now)
train_events = list(filter(lambda x: x['phase'] == 'Train', events_obj))
valid_events = list(filter(lambda x: x['phase'] == 'Valid', events_obj))
test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj))
post = {
"dataset": dataset,
"config": config_obj,
"train_events": train_events,
"valid_events": valid_events,
"test_events": test_events,
"username": username,
"hostname": hostname,
"date": date,
"label": label,
"sha1": config_sha1,
"version": version,
"checkpoint": checkpoint
}
if 'eid' in kwargs:
post.update({'_id': ObjectId(kwargs['eid'])})
try:
coll = self.db[task]
result = coll.insert_one(post)
return BackendSuccess(message=str(result.inserted_id))
except pymongo.errors.PyMongoError as e:
return BackendError(message='experiment could not be inserted: {}'.format(e.message))
def launch(
config, settings,
logging, hpctl_logging,
datasets, embeddings,
reporting, unknown,
task, num_iters, **kwargs
):
mead_config = get_config(config, reporting, unknown)
hp_settings, mead_settings = get_settings(settings)
load_user_modules(mead_config, hp_settings)
exp_hash = hash_config(mead_config)
hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging)
datasets = read_config_file_or_json(datasets)
embeddings = read_config_file_or_json(embeddings)
if task is None:
task = mead_config.get('task', 'classify')
_, backend_config = get_ends(hp_settings, unknown)
force_remote_backend(backend_config)
config_sampler = get_config_sampler(mead_config, None)
be = get_backend(backend_config)
for _ in range(num_iters):
label, config = config_sampler.sample()
print(label)
send = {
'label': label,
'config': config,
'mead_logs': mead_logs,
'hpctl_logs': hp_logs,
'task_name': task,
'settings': mead_settings,
'experiment_config': mead_config,
}
be.launch(**send)
def test_experiment_details(setup):
date = datetime.datetime.utcnow().isoformat()
label = 'test_label'
dataset = 'test_dataset'
config = '{"test":"test"}'
username = '******'
hostname = 'host'
exp = Experiment(dataset=dataset,
label=label,
exp_date=date,
config=config,
username=username,
hostname=hostname,
train_events=[],
valid_events=[],
test_events=[
Result(metric='t',
tick_type='t',
phase='Test',
tick=0,
value=0.1)
])
eid = _put_one_exp(exp)
try:
result = API.list_experiments_by_prop(TASK, eid=eid)
except ApiException:
print(eid)
return False
exp = result[0]
assert exp.dataset == 'test_dataset'
assert exp.label == label
assert (exp.exp_date == date) or (exp.exp_date[:-1] == date
) # sql inserts Z at the end
assert exp.sha1 == hash_config(json.loads('{"test": "test"}'))
assert exp.username == username
assert exp.hostname == hostname
def search(
config, settings,
logging, hpctl_logging,
datasets, embeddings,
reporting, unknown,
task, num_iters,
**kwargs
):
"""Search for optimal hyperparameters."""
mead_config = get_config(config, reporting, unknown)
hp_settings, mead_settings = get_settings(settings)
load_user_modules(mead_config, hp_settings)
exp_hash = hash_config(mead_config)
hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging)
datasets = read_config_file_or_json(datasets)
embeddings = read_config_file_or_json(embeddings)
if task is None:
task = mead_config.get('task', 'classify')
frontend_config, backend_config = get_ends(hp_settings, unknown)
# Figure out xpctl
xpctl_config = None
auto_xpctl = 'xpctl' in mead_config.get('reporting', [])
if not auto_xpctl:
# If the jobs aren't setup to use xpctl automatically create your own
xpctl_config = get_xpctl_settings(mead_settings)
if xpctl_config is not None:
xpctl_extra = parse_extra_args(['xpctl'], unknown)
xpctl_config['label'] = xpctl_extra.get('xpctl', {}).get('label')
results_config = {}
# Set frontend defaults
frontend_config['experiment_hash'] = exp_hash
default = mead_config['train'].get('early_stopping_metric', 'avg_loss')
frontend_config.setdefault('train', 'avg_loss')
frontend_config.setdefault('dev', default)
frontend_config.setdefault('test', default)
# Negotiate remote status
if backend_config['type'] != 'remote':
set_root(hp_settings)
_remote_monkey_patch(backend_config, hp_logs, results_config, xpctl_config)
xpctl = get_xpctl(xpctl_config)
results = get_results(results_config)
results.add_experiment(mead_config)
backend = get_backend(backend_config)
config_sampler = get_config_sampler(mead_config, results)
logs = get_log_server(hp_logs)
frontend = get_frontend(frontend_config, results, xpctl)
labels = run(num_iters, results, backend, frontend, config_sampler, logs, mead_logs, hp_logs, mead_settings, datasets, embeddings, task)
logs.stop()
frontend.finalize()
results.save()
if auto_xpctl:
for label in labels:
results.set_xpctl(label, True)
return labels, results
def add_experiment(self, exp_config):
exp_hash = hash_config(exp_config)
self.exp_to_config[exp_hash] = exp_config
def test_list_experiments_by_prop(setup):
"""
list_experiments_by_prop finds some experiments by a (prop, value) pair. also allows some filtering:
:param setup:
:return:
"""
configs = ['{"c1":"c1"}', '{"c2":"c2"}']
datasets = ['d1', 'd2']
labels = ['l1', 'l2']
users = ['u1', 'u2', 'u3']
metrics = ['f1', 'acc', 'random']
test_events = [
Result(metric=metric,
value=0.5,
tick_type='EPOCH',
tick=0,
phase='Test') for metric in metrics
]
experiments = []
exp_detail = namedtuple('exp_detail',
['eid', 'sha1', 'dataset', 'label', 'username'])
for config in configs:
for dataset in datasets:
for label in labels:
for username in users:
result = _put_one_exp(
Experiment(config=config,
dataset=dataset,
label=label,
username=username,
train_events=[],
valid_events=[],
test_events=test_events))
experiments.append(
exp_detail(eid=result,
sha1=hash_config(json.loads(config)),
dataset=dataset,
label=label,
username=username))
# find by a property and group by different reduction dims
for prop, values in {
'dataset': datasets,
'label': labels,
'username': users
}.items():
for value in values:
assert _test_list_experiments_by_prop(prop=prop,
value=value,
experiments=experiments)
# test the `users` filter work
prop = 'dataset'
value = 'd1'
users = [['u1', 'u2'], ['u1']]
for user in users:
results = API.list_experiments_by_prop(TASK, dataset=value, user=user)
result_eids = [x.eid for x in results]
expected_eids = [
x.eid for x in experiments
if x.dataset == 'd1' and x.username in user
]
assert set(result_eids) == set(expected_eids)
# test sort works
config = configs[0]
metrics = ['f1', 'acc']
exp_value = namedtuple('exp_value', ['eid', 'value'])
experiments = []
dataset = _generate_random_string() # generate a unique dataset
for value in [0.5, 0.7, 0.8]:
test_events = [
Result(metric=metric,
value=value,
tick_type='EPOCH',
tick=0,
phase='Test') for metric in metrics
]
result = _put_one_exp(
Experiment(config=config,
train_events=[],
valid_events=[],
test_events=test_events,
dataset=dataset))
experiments.append(exp_value(eid=result, value=value))
results = API.list_experiments_by_prop(
TASK, dataset=dataset, sort='f1') # get results only from that
# unique dataset, sort by f1
_max = experiments[-1]
assert results[0].eid == _max.eid
def search(config, settings, logging, hpctl_logging, datasets, embeddings,
reporting, unknown, task, num_iters, **kwargs):
"""Search for optimal hyperparameters."""
mead_config = get_config(config, reporting, unknown)
hp_settings, mead_settings = get_settings(settings)
load_user_modules(mead_config, hp_settings)
exp_hash = hash_config(mead_config)
hp_logs, mead_logs = get_logs(hp_settings, logging, hpctl_logging)
datasets = read_config_file_or_json(datasets)
embeddings = read_config_file_or_json(embeddings)
if task is None:
task = mead_config.get('task', 'classify')
frontend_config, backend_config = get_ends(hp_settings, unknown)
# Figure out xpctl
xpctl_config = None
auto_xpctl = 'xpctl' in mead_config.get('reporting', [])
if not auto_xpctl:
# If the jobs aren't setup to use xpctl automatically create your own
xpctl_config = get_xpctl_settings(mead_settings)
if xpctl_config is not None:
xpctl_extra = parse_extra_args(['xpctl'], unknown)
xpctl_config['label'] = xpctl_extra.get('xpctl', {}).get('label')
results_config = {}
# Set frontend defaults
frontend_config['experiment_hash'] = exp_hash
default = mead_config['train'].get('early_stopping_metric', 'avg_loss')
frontend_config.setdefault('train', 'avg_loss')
frontend_config.setdefault('dev', default)
frontend_config.setdefault('test', default)
# Negotiate remote status
if backend_config['type'] != 'remote':
set_root(hp_settings)
_remote_monkey_patch(backend_config, hp_logs, results_config, xpctl_config)
xpctl = get_xpctl(xpctl_config)
results = get_results(results_config)
results.add_experiment(mead_config)
backend = get_backend(backend_config)
config_sampler = get_config_sampler(mead_config, results)
logs = get_log_server(hp_logs)
frontend = get_frontend(frontend_config, results, xpctl)
labels = run(num_iters, results, backend, frontend, config_sampler, logs,
mead_logs, hp_logs, mead_settings, datasets, embeddings, task)
logs.stop()
frontend.finalize()
results.save()
if auto_xpctl:
for label in labels:
results.set_xpctl(label, True)
return labels, results
