def started_event(self, ex_info, command, host_info, start_time, config,
meta_info, _id):
if self.overwrite is None:
self.run_entry = {'_id': _id}
else:
if self.run_entry is not None:
raise RuntimeError("Cannot overwrite more than once!")
# sanity checks
if self.overwrite['experiment']['sources'] != ex_info['sources']:
raise RuntimeError("Sources don't match")
self.run_entry = self.overwrite
self.run_entry.update({
'experiment': dict(ex_info),
'format': self.VERSION,
'command': command,
'host': dict(host_info),
'start_time': start_time,
'config': flatten(config),
'meta': meta_info,
'status': 'RUNNING',
'resources': [],
'artifacts': [],
'captured_out': '',
'info': {},
'heartbeat': None
})
# save sources
self.run_entry['experiment']['sources'] = self.save_sources(ex_info)
self.insert()
return self.run_entry['_id']
def save_config(_config, _log, config_filename='config.json'):
"""
Store the updated configuration in a file.
By default uses the filename "config.json", but that can be changed by
setting the config_filename config entry.
"""
if 'config_filename' in _config:
del _config['config_filename']
_log.info('Saving config to "{}"'.format(config_filename))
save_config_file(flatten(_config), config_filename)
def save_config(_config, _log, config_filename='config.json'):
"""
Store the updated configuration in a file.
By default uses the filename "config.json", but that can be changed by
setting the config_filename config entry.
"""
if 'config_filename' in _config:
del _config['config_filename']
_log.info('Saving config to "{}"'.format(config_filename))
save_config_file(flatten(_config), config_filename)
def save_config(_config, _log, config_filename="config.json"):
"""
Store the updated configuration in a file.
By default uses the filename "config.json", but that can be changed by
setting the config_filename config entry.
"""
# Copy the config to make it mutable
_config = copy.deepcopy(_config)
if "config_filename" in _config:
del _config["config_filename"]
_log.info('Saving config to "{}"'.format(config_filename))
save_config_file(flatten(_config), config_filename)
def queued_event(self, ex_info, command, queue_time, config, meta_info,
_id):
if self.overwrite is not None:
raise RuntimeError("Can't overwrite with QUEUED run.")
self.run_entry = {
'experiment': dict(ex_info),
'command': command,
'config': flatten(config),
'meta': meta_info,
'status': 'QUEUED'
}
# set ID if given
if _id is not None:
self.run_entry['_id'] = _id
# save sources
self.run_entry['experiment']['sources'] = self.save_sources(ex_info)
self.final_save(attempts=1)
return self.run_entry['_id']
def queued_event(self, ex_info, command, queue_time, config, meta_info,
_id):
Base.metadata.create_all(self.engine)
sql_exp = Experiment.get_or_create(ex_info, self.session)
if _id is None:
i = self.session.query(Run).order_by(Run.run_id.desc()).first()
_id = '0' if i is None else str(int(i.id) + 1)
self.run = Run(run_id=_id,
config=json.dumps(flatten(config)),
command=command,
priority=meta_info.get('priority', 0),
comment=meta_info.get('comment', ''),
experiment=sql_exp,
status='QUEUED')
self.session.add(self.run)
self.session.commit()
return _id or self.run.run_id
def queued_event(self, ex_info, command, host_info, queue_time, config,
meta_info, _id):
if self.overwrite is not None:
raise RuntimeError("Can't overwrite with QUEUED run.")
self.run_entry = {
'experiment': dict(ex_info),
'command': command,
'host': dict(host_info),
'config': flatten(config),
'meta': meta_info,
'status': 'QUEUED'
}
# set ID if given
if _id is not None:
self.run_entry['_id'] = _id
# save sources
self.run_entry['experiment']['sources'] = self.save_sources(ex_info)
self.insert()
return self.run_entry['_id']
def queued_event(self, ex_info, command, host_info, queue_time, config,
meta_info, _id):
if self.overwrite is not None:
raise RuntimeError("Can't overwrite with QUEUED run.")
self.run_entry = {
"experiment": dict(ex_info),
"command": command,
"host": dict(host_info),
"config": flatten(config),
"meta": meta_info,
"status": "QUEUED",
}
# set ID if given
if _id is not None:
self.run_entry["_id"] = _id
# save sources
self.run_entry["experiment"]["sources"] = self.save_sources(ex_info)
self.insert()
return self.run_entry["_id"]
def queued_event(self, ex_info, command, host_info, queue_time, config,
meta_info, _id):
Base.metadata.create_all(self.engine)
sql_exp = Experiment.get_or_create(ex_info, self.session)
sql_host = Host.get_or_create(host_info, self.session)
if _id is None:
i = self.session.query(Run).order_by(Run.id.desc()).first()
_id = 0 if i is None else i.id + 1
self.run = Run(run_id=str(_id),
config=json.dumps(flatten(config)),
command=command,
priority=meta_info.get('priority', 0),
comment=meta_info.get('comment', ''),
experiment=sql_exp,
host=sql_host,
status='QUEUED')
self.session.add(self.run)
self.save()
return _id or self.run.run_id
def started_event(self, ex_info, command, host_info, start_time, config,
meta_info, _id):
Base.metadata.create_all(self.engine)
sql_exp = Experiment.get_or_create(ex_info, self.session)
sql_host = Host.get_or_create(host_info, self.session)
if _id is None:
i = self.session.query(Run).order_by(Run.id.desc()).first()
_id = 0 if i is None else i.id + 1
self.run = Run(run_id=str(_id),
start_time=start_time,
config=json.dumps(flatten(config)),
command=command,
priority=meta_info.get('priority', 0),
comment=meta_info.get('comment', ''),
experiment=sql_exp,
host=sql_host,
status='RUNNING')
self.session.add(self.run)
self.save()
return _id or self.run.run_id
__all__ = ('load_config_file', 'save_config_file')
class Handler(object):
def __init__(self, load, dump, mode):
self.load = load
self.dump = dump
self.mode = mode
HANDLER_BY_EXT = {
'.json':
Handler(
lambda fp: restore(json.load(fp)),
lambda obj, fp: json.dump(flatten(obj), fp, sort_keys=True, indent=2),
''),
'.pickle':
Handler(pickle.load, pickle.dump, 'b'),
}
if opt.has_yaml:
HANDLER_BY_EXT['.yaml'] = Handler(opt.yaml.load, opt.yaml.dump, '')
HANDLER_BY_EXT['.yml'] = Handler(opt.yaml.load, opt.yaml.dump, '')
def get_handler(filename):
_, extension = os.path.splitext(filename)
return HANDLER_BY_EXT[extension]
def test_restore_on_json_is_noop(obj):
assert flatten(obj) == obj
def heartbeat_event(self, info, captured_out, beat_time, result):
self.run.info = json.dumps(flatten(info))
self.run.captured_out = captured_out
self.run.heartbeat = beat_time
self.run.result = result
self.save()
def test_serialize_pandas_dataframes():
pd, np = opt.pandas, opt.np
df = pd.DataFrame(np.arange(20).reshape(5, 4), columns=list("ABCD"))
b = restore(flatten(df))
assert np.all(df == b)
assert np.all(df.dtypes == b.dtypes)
def test_serialize_pandas_dataframes():
pd, np = opt.pandas, opt.np
df = pd.DataFrame(np.arange(20).reshape(5, 4), columns=list('ABCD'))
b = restore(flatten(df))
assert np.all(df == b)
assert np.all(df.dtypes == b.dtypes)
def save_json(self, obj, filename):
with open(os.path.join(self.dir, filename), 'w') as f:
json.dump(flatten(obj), f, sort_keys=True, indent=2)
def test_serialize_numpy_arrays():
a = opt.np.array([[1, 2, 3], [4, 5, 6]], dtype=opt.np.float32)
b = restore(flatten(a))
assert opt.np.all(b == a)
assert b.dtype == a.dtype
assert b.shape == a.shape
def completed_event(self, stop_time, result):
self.run_entry["stop_time"] = stop_time
self.run_entry["result"] = flatten(result)
self.run_entry["status"] = "COMPLETED"
self.final_save(attempts=10)
def completed_event(self, stop_time, result):
self.run_entry['stop_time'] = stop_time
self.run_entry['result'] = flatten(result)
self.run_entry['status'] = 'COMPLETED'
self.final_save(attempts=10)
def completed_event(self, stop_time, result):
self.run_entry['stop_time'] = stop_time
self.run_entry['result'] = flatten(result)
self.run_entry['status'] = 'COMPLETED'
self.final_save(attempts=10)
__sacred__ = True # marks files that should be filtered from stack traces
__all__ = ('load_config_file', 'save_config_file')
class Handler(object):
def __init__(self, load, dump, mode):
self.load = load
self.dump = dump
self.mode = mode
HANDLER_BY_EXT = {
'.json': Handler(lambda fp: restore(json.load(fp)),
lambda obj, fp: json.dump(flatten(obj), fp,
sort_keys=True, indent=2), ''),
'.pickle': Handler(pickle.load, pickle.dump, 'b'),
}
if opt.has_yaml:
HANDLER_BY_EXT['.yaml'] = Handler(opt.yaml.load, opt.yaml.dump, '')
def get_handler(filename):
_, extension = os.path.splitext(filename)
return HANDLER_BY_EXT[extension]
def load_config_file(filename):
def heartbeat_event(self, info, captured_out, beat_time, result):
self.run_entry['info'] = flatten(info)
self.run_entry['captured_out'] = captured_out
self.run_entry['heartbeat'] = beat_time
self.run_entry['result'] = flatten(result)
self.save()
def test_serialize_non_str_keys():
d = {1: "one", 2: "two"}
assert restore(flatten(d)) == d
def test_flatten_normalizes_numpy_scalars(typename):
dtype = getattr(opt.np, typename)
a = 1
b = flatten(dtype(a))
assert a == b
assert isinstance(b, (int, bool, float))
def test_serialize_numpy_arrays():
a = opt.np.array([[1, 2, 3], [4, 5, 6]], dtype=opt.np.float32)
b = restore(flatten(a))
assert opt.np.all(b == a)
assert b.dtype == a.dtype
assert b.shape == a.shape
def test_restore_on_json_is_noop(obj):
assert flatten(obj) == obj
def heartbeat_event(self, info, captured_out, beat_time, result):
self.run_entry["info"] = flatten(info)
self.run_entry["captured_out"] = captured_out
self.run_entry["heartbeat"] = beat_time
self.run_entry["result"] = flatten(result)
self.save()
def test_flatten_normalizes_numpy_scalars(typename):
dtype = getattr(opt.np, typename)
a = 1
b = flatten(dtype(a))
assert a == b
assert isinstance(b, (int, bool, float))
def heartbeat_event(self, info, captured_out, beat_time, result):
self.run_entry['info'] = flatten(info)
self.run_entry['captured_out'] = captured_out
self.run_entry['heartbeat'] = beat_time
self.run_entry['result'] = flatten(result)
self.save()
def heartbeat_event(self, info, captured_out, beat_time, result):
self.run.info = json.dumps(flatten(info))
self.run.captured_out = captured_out
self.run.heartbeat = beat_time
self.run.result = result
self.save()
def test_flatten_on_json_is_noop(obj):
assert flatten(obj) == obj
def test_flatten_on_json_is_noop(obj):
assert flatten(obj) == obj
def test_serialize_non_str_keys():
d = {1: "one", 2: "two"}
assert restore(flatten(d)) == d
def heartbeat_event(self, info, captured_out, beat_time):
self.run.info = json.dumps(flatten(info))
self.run.captured_out = captured_out
self.run.heartbeat = beat_time
self.session.commit()
def test_serialize_tuples():
t = (1, "two")
assert restore(flatten(t)) == t
assert isinstance(restore(flatten(t)), tuple)
def save_json(self, obj, filename):
with open(os.path.join(self.dir, filename), "w") as f:
json.dump(flatten(obj), f, sort_keys=True, indent=2)
def save_json(self, obj, filename):
key = s3_join(self.dir, filename)
self.put_data(key, json.dumps(flatten(obj), sort_keys=True, indent=2))
def test_serialize_tuples():
t = (1, 'two')
assert restore(flatten(t)) == t
assert isinstance(restore(flatten(t)), tuple)
