def get_model_location(self, task, eid):
coll = self.db[task]
query = {'_id': ObjectId(eid)}
projection = {'checkpoint': 1}
results = [x.get('checkpoint') for x in list(coll.find(query, projection))]
results = [x for x in results if x is not None]
if not results:
return BackendError(message='no model location for experiment id [{}] in [{}] database'.format(eid, task))
return BackendSuccess(results[0])
def update_prop(self, task, eid, prop, value):
try:
coll = self.db[task]
r = coll.find_one({'_id': ObjectId(eid)}, {prop: 1})
if r is None:
return BackendError(message='property {} for experiment {} not found in {} database'.format(prop, eid, task))
prev_value = r[prop]
coll.update({'_id': ObjectId(eid)}, {'$set': {prop: value}}, upsert=False)
changed_value = coll.find_one({'_id': ObjectId(eid)}, {prop: 1})[prop]
return BackendSuccess(message='for experiment [{}] property [{}] was changed from [{}] to [{}]'
.format(eid, prop, prev_value, changed_value))
except pymongo.errors.PyMongoError as e:
return BackendError(message='property update failed: {}'.format(e.message))
def update_prop(self, task, eid, prop, value):
try:
session = self.Session()
exp = session.query(SqlExperiment).filter(SqlExperiment.task == task).filter(SqlExperiment.eid == eid)
if exp is None or exp.scalar() is None:
return BackendError(message='no experiment with id [{}] for task [{}]'.format(eid, task))
# there must be a better way of getting a column value through a column name
prev_value = getattr(session.query(SqlExperiment).get(eid), prop)
session.query(SqlExperiment).filter(SqlExperiment.eid == eid).update({prop: value})
session.commit()
changed_value = getattr(session.query(SqlExperiment).get(eid), prop)
return BackendSuccess(message='for experiment [{}] property [{}] was changed from [{}] to [{}].'
.format(eid, prop, prev_value, changed_value))
except sql.exc.SQLAlchemyError as e:
return BackendError(message=str(e))
def _put_result(self, task, config_obj, events_obj, **kwargs):
session = self.Session()
now = safe_get(kwargs, 'date', datetime.datetime.utcnow().isoformat())
hostname = safe_get(kwargs, 'hostname', socket.gethostname())
username = safe_get(kwargs, 'username', getpass.getuser())
config_sha1 = safe_get(kwargs, 'sha1', hash_config(config_obj))
label = safe_get(kwargs, 'label', get_experiment_label(config_obj, task, **kwargs))
checkpoint = kwargs.get('checkpoint')
version = safe_get(kwargs, 'version', __version__)
dataset = safe_get(kwargs, 'dataset', config_obj.get('dataset'))
date = safe_get(kwargs, 'exp_date', now)
events = []
for event in events_obj:
tick = event['tick']
phase = event['phase']
event_obj = SqlEvent(phase=phase, tick_type=event['tick_type'], tick=tick, results=[])
for key in event.keys():
if key not in ['tick_type', 'tick', 'event_type', 'id', 'date', 'phase']:
metric = SqlResult(metric=key, value=event[key])
event_obj.results += [metric]
events += [event_obj]
experiment = SqlExperiment(
label=label,
checkpoint=checkpoint,
sha1=config_sha1,
task=task,
dataset=dataset,
config=json.dumps(config_obj),
hostname=hostname,
username=username,
date=date,
version=version,
status='CREATED',
last_modified=now,
events=events
)
try:
session.add(experiment)
session.commit()
return BackendSuccess(message=experiment.eid)
except sql.exc.SQLAlchemyError as e:
return BackendError(message=str(e))
def remove_experiment(self, task, eid):
try:
coll = self.db[task]
prev = coll.find_one({'_id': ObjectId(eid)})
if prev is None:
return BackendError(message='delete operation failed: experiment [{}] not found in [{}] database'.format(eid, task))
model_loc_response = self.get_model_location(task, eid)
model_loc = model_loc_response.message
if model_loc is not None and type(model_loc_response) is not BackendError and os.path.exists(model_loc):
try:
os.remove(model_loc)
except IOError:
return BackendError(message='model {} exists on host but could not be removed'.format(model_loc))
coll.remove({'_id': ObjectId(eid)})
try:
assert coll.find_one({'_id': ObjectId(eid)}) is None
return BackendSuccess("record [{}] deleted successfully from database [{}]".format(eid, task))
except AssertionError:
return BackendError('delete failed: could not delete experiment {} from {} database'.format(eid, task))
except pymongo.errors.PyMongoError as e:
return BackendError(message='experiment could not be removed: {}'.format(e.message))
def remove_experiment(self, task, eid):
try:
session = self.Session()
exp = session.query(SqlExperiment).filter(SqlExperiment.task == task).filter(SqlExperiment.eid == eid)
if exp is None or exp.scalar() is None:
return BackendError(message='no experiment with id [{}] for task [{}]'.format(eid, task))
model_loc_response = self.get_model_location(task, eid)
model_loc = model_loc_response.message
if model_loc is not None and type(model_loc_response) is not BackendError and os.path.exists(model_loc):
try:
os.remove(model_loc)
except IOError:
return BackendError(message='model {} exists on host but could not be removed'.format(model_loc))
exp = session.query(SqlExperiment).get(eid)
session.delete(exp)
session.commit()
try:
assert session.query(SqlExperiment).get(eid) is None
return BackendSuccess("record [{}] deleted successfully from database [{}]".format(eid, task))
except AssertionError:
return BackendError('delete failed: could not delete experiment {} from {} database'.format(eid, task))
except sql.exc.SQLAlchemyError as e:
return BackendError(message=str(e))
def _put_result(self, task, config_obj, events_obj, **kwargs):
now = safe_get(kwargs, 'date', datetime.datetime.utcnow().isoformat())
hostname = safe_get(kwargs, 'hostname', socket.gethostname())
username = safe_get(kwargs, 'username', getpass.getuser())
config_sha1 = safe_get(kwargs, 'sha1', hash_config(config_obj))
label = safe_get(kwargs, 'label', get_experiment_label(config_obj, task, **kwargs))
checkpoint = kwargs.get('checkpoint')
version = safe_get(kwargs, 'version', __version__)
dataset = safe_get(kwargs, 'dataset', config_obj.get('dataset'))
date = safe_get(kwargs, 'exp_date', now)
train_events = list(filter(lambda x: x['phase'] == 'Train', events_obj))
valid_events = list(filter(lambda x: x['phase'] == 'Valid', events_obj))
test_events = list(filter(lambda x: x['phase'] == 'Test', events_obj))
post = {
"dataset": dataset,
"config": config_obj,
"train_events": train_events,
"valid_events": valid_events,
"test_events": test_events,
"username": username,
"hostname": hostname,
"date": date,
"label": label,
"sha1": config_sha1,
"version": version,
"checkpoint": checkpoint
}
if 'eid' in kwargs:
post.update({'_id': ObjectId(kwargs['eid'])})
try:
coll = self.db[task]
result = coll.insert_one(post)
return BackendSuccess(message=str(result.inserted_id))
except pymongo.errors.PyMongoError as e:
return BackendError(message='experiment could not be inserted: {}'.format(e.message))
def get_model_location(self, task, eid):
session = self.Session()
exp = session.query(SqlExperiment).get(eid)
return BackendSuccess(exp.checkpoint)