def put_result(self, task, config_obj, events_obj, **kwargs):
now = datetime.datetime.utcnow().isoformat()
train_events = list(filter(lambda x: x['phase'] == 'Train',
events_obj))
valid_events = list(filter(lambda x: x['phase'] == 'Valid',
events_obj))
test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj))
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
print_fn = kwargs.get('print_fn', print)
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hashlib.sha1(
json.dumps(order_json(config_obj)).encode('utf-8')).hexdigest()
label = kwargs.get("label", config_sha1)
post = {
"config": config_obj,
"train_events": train_events,
"valid_events": valid_events,
"test_events": test_events,
"username": username,
"hostname": hostname,
"date": now,
"label": label,
"sha1": config_sha1,
"version": __version__
}
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1,
checkpoint_store)
if model_loc is not None:
post.update({
"checkpoint":
"{}:{}".format(hostname, os.path.abspath(model_loc))
})
else:
print_fn("model could not be stored, see previous errors")
if task in self.db.collection_names():
print_fn(
"updating results for existing task [{}] in host [{}]".format(
task, self.dbhost))
else:
print_fn("creating new task [{}] in host [{}]".format(
task, self.dbhost))
coll = self.db[task]
result = coll.insert_one(post)
print_fn(
"results updated, the new results are stored with the record id: {}"
.format(result.inserted_id))
return result.inserted_id
def put_model(self, id, task, checkpoint_base, checkpoint_store, print_fn=print):
session = self.Session()
exp = session.query(Experiment).get(id)
if exp is None:
print_fn("no sha1 for the given id found, returning.")
return None
sha1 = exp.sha1
model_loc = store_model(checkpoint_base, sha1, checkpoint_store, print_fn)
if model_loc is not None:
exp.checkpoint = model_loc
session.commit()
return model_loc
def put_result(self, task, config_obj, events_obj, **kwargs):
session = self.Session()
print_fn = kwargs.get('print_fn', print)
now = datetime.datetime.utcnow().isoformat()
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hashlib.sha1(
json.dumps(config_obj).encode('utf-8')).hexdigest()
label = kwargs.get("label", config_sha1)
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
checkpoint = None
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1,
checkpoint_store)
if model_loc is not None:
checkpoint = "{}:{}".format(hostname,
os.path.abspath(model_loc))
else:
print_fn("model could not be stored, see previous errors")
event_objs = []
for event in events_obj:
tick = event['tick']
date = event.get('date')
phase = event['phase']
event_obj = Event(tick=tick, date=date, phase=phase, metrics=[])
for key in event.keys():
if key not in [
'tick_type', 'tick', 'event_type', 'id', 'date',
'phase'
]:
metric = Metric(label=key, value=event[key])
event_obj.metrics += [metric]
event_objs += [event_obj]
experiment = Experiment(label=label,
checkpoint=checkpoint,
sha1=config_sha1,
task=task,
dataset=config_obj['dataset'],
config=json.dumps(config_obj),
hostname=hostname,
username=username,
date=now,
version=__version__,
status='CREATED',
last_modified=now)
experiment.events = event_objs
session.add(experiment)
session.commit()
return experiment
def put_model(self, id, task, checkpoint_base, checkpoint_store, print_fn=print):
coll = self.db[task]
query = {'_id': ObjectId(id)}
projection = {'sha1': 1}
results = list(coll.find(query, projection))
if not results:
print_fn("no sha1 for the given id found, returning.")
return None
sha1 = results[0]['sha1']
model_loc = store_model(checkpoint_base, sha1, checkpoint_store, print_fn)
if model_loc is not None:
coll.update_one({'_id': ObjectId(id)}, {'$set': {'checkpoint': model_loc}}, upsert=False)
return model_loc
def put_model(self, id, task, checkpoint_base, checkpoint_store, print_fn=print):
coll = self.db[task]
query = {'_id': ObjectId(id)}
projection = {'sha1': 1}
results = list(coll.find(query, projection))
if not results:
print_fn("no sha1 for the given id found, returning.")
return False
sha1 = results[0]['sha1']
model_loc = store_model(checkpoint_base, sha1, checkpoint_store, print_fn)
if model_loc is not None:
coll.update_one({'_id': ObjectId(id)}, {'$set': {'checkpoint': model_loc}}, upsert=False)
return model_loc
def put_result(self, task, config_obj, events_obj, **kwargs):
session = self.Session()
print_fn = kwargs.get('print_fn', print)
now = datetime.datetime.utcnow().isoformat()
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hash_config(config_obj)
label = get_experiment_label(config_obj, task, **kwargs)
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
checkpoint = None
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store)
if model_loc is not None:
checkpoint = "{}:{}".format(hostname, os.path.abspath(model_loc))
else:
print_fn("model could not be stored, see previous errors")
event_objs = []
for event in events_obj:
tick = event['tick']
date = event.get('date')
phase = event['phase']
event_obj = Event(tick=tick, date=date, phase=phase, metrics=[])
for key in event.keys():
if key not in ['tick_type', 'tick', 'event_type', 'id', 'date', 'phase']:
metric = Metric(label=key, value=event[key])
event_obj.metrics += [metric]
event_objs += [event_obj]
experiment = Experiment(
label=label,
checkpoint=checkpoint,
sha1=config_sha1,
task=task,
dataset=config_obj['dataset'],
config=json.dumps(config_obj),
hostname=hostname,
username=username,
date=now,
version=__version__,
status='CREATED',
last_modified=now
)
experiment.events = event_objs
session.add(experiment)
session.commit()
return experiment
def put_model(self,
id,
task,
checkpoint_base,
checkpoint_store,
print_fn=print):
session = self.Session()
exp = session.query(Experiment).get(id)
if exp is None:
print_fn("no sha1 for the given id found, returning.")
return None
sha1 = exp.sha1
model_loc = store_model(checkpoint_base, sha1, checkpoint_store,
print_fn)
if model_loc is not None:
exp.checkpoint = model_loc
session.commit()
return model_loc
def put_result(self, task, config_obj, events_obj, **kwargs):
now = datetime.datetime.utcnow().isoformat()
train_events = list(filter(lambda x: x['phase'] == 'Train', events_obj))
valid_events = list(filter(lambda x: x['phase'] == 'Valid', events_obj))
test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj))
checkpoint_base = kwargs.get('checkpoint_base', None)
checkpoint_store = kwargs.get('checkpoint_store', None)
print_fn = kwargs.get('print_fn', print)
hostname = kwargs.get('hostname', socket.gethostname())
username = kwargs.get('username', getpass.getuser())
config_sha1 = hash_config(config_obj)
label = get_experiment_label(config_obj, task, **kwargs)
post = {
"config": config_obj,
"train_events": train_events,
"valid_events": valid_events,
"test_events": test_events,
"username": username,
"hostname": hostname,
"date": now,
"label": label,
"sha1": config_sha1,
"version": __version__
}
if checkpoint_base:
model_loc = store_model(checkpoint_base, config_sha1, checkpoint_store)
if model_loc is not None:
post.update({"checkpoint": "{}:{}".format(hostname, os.path.abspath(model_loc))})
else:
print_fn("model could not be stored, see previous errors")
if task in self.db.collection_names():
print_fn("updating results for existing task [{}] in host [{}]".format(task, self.dbhost))
else:
print_fn("creating new task [{}] in host [{}]".format(task, self.dbhost))
coll = self.db[task]
result = coll.insert_one(post)
print_fn("results updated, the new results are stored with the record id: {}".format(result.inserted_id))
return result.inserted_id