def add_group_trial(self, group: TrialGroup, trial: Trial):
kwargs = dict()
kwargs['__rpc__'] = 'add_group_trial'
kwargs['group'] = to_json(group)
kwargs['trial'] = to_json(trial)
send(self.socket, kwargs)
return _check(recv(self.socket))
def add_project_trial(self, project: Project, trial: Trial):
kwargs = dict()
kwargs['__rpc__'] = 'add_project_trial'
kwargs['project'] = to_json(project)
kwargs['trial'] = to_json(trial)
send(self.socket, kwargs)
return _check(recv(self.socket))
def register_trial(self, trial):
"""Create a new trial to be executed"""
stamp = datetime.datetime.utcnow()
trial.submit_time = stamp
metadata = dict()
# pylint: disable=protected-access
metadata["params_types"] = {
remove_leading_slash(p.name): p.type
for p in trial._params
}
metadata["submit_time"] = to_json(trial.submit_time)
metadata["end_time"] = to_json(trial.end_time)
metadata["worker"] = trial.worker
metadata["metric_types"] = {
remove_leading_slash(p.name): p.type
for p in trial.results
}
metadata["metric_types"][self.objective] = "objective"
heartbeat = to_json(trial.heartbeat)
if heartbeat is None:
heartbeat = 0
metadata["heartbeat"] = heartbeat
metrics = defaultdict(list)
for p in trial.results:
metrics[p.name] = [p.value]
if self.project is None:
self._get_project(self.group.project_id)
trial = self.backend.new_trial(
TrackTrial(
_hash=trial.hash_name,
status=get_track_status(trial.status),
project_id=self.project.uid,
group_id=self.group.uid,
parameters=trial.params,
metadata=metadata,
metrics=metrics,
),
auto_increment=False,
)
if trial is None:
raise DuplicateKeyError("Was not able to register Trial!")
return TrialAdapter(trial, objective=self.objective)
def new_project(self, project: Project):
kwargs = dict()
kwargs['__rpc__'] = 'new_project'
kwargs['project'] = to_json(project)
send(self.socket, kwargs)
p = _check(recv(self.socket))
return p
def new_project(self, project: Project):
try:
self.client.write('projects', to_json(project))
return project
except DuplicateKeyError:
return self.get_project(project)
def set_trial_status(self, trial: Trial, status, error=None):
self.client.write('trials',
query={'uid': trial.uid},
data={'$set': {
'status': to_json(status)
}})
return self.check_result()
def set_trial_status(self, trial: Trial, status, error=None):
kwargs = dict()
kwargs['__rpc__'] = 'set_trial_status'
kwargs['trial'] = trial.uid
kwargs['status'] = to_json(status)
send(self.socket, kwargs)
return _check(recv(self.socket))
def new_trial_group(self, group: TrialGroup):
try:
self.client.write('groups', to_json(group))
return group
except DuplicateKeyError:
return self.get_trial_group(group)
def new_project(self, project: Project):
try:
self.projects.insert_one(
to_json(project)
)
return project
except DuplicateKeyError:
return self.get_project(project)
def new_trial(self, trial: Trial, auto_increment=None):
try:
self.trials.insert_one(to_json(trial))
return trial
except DuplicateKeyError:
trial.revision += 1
info(f'Trial already exist increasing revision (rev: {trial.revision})')
return self.new_trial(trial)
def fetch_and_update_trial(self, query, attr, *args, **kwargs):
if attr == 'set_trial_status':
return from_json(
self.groups.find_one_and_update(
query,
{'$set': {'status': to_json(kwargs['status'])}}, return_document=pymongo.ReturnDocument.AFTER),
dtype='trial')
raise NotImplementedError()
def test_trial_group():
p = TrialGroup(name='1',
description='2',
metadata=dict(a=2, b=3),
trials=set(),
project_id=1)
ps = from_json(to_json(p))
assert p == ps
def test_project():
p = Project(name='1',
description='2',
metadata=dict(a=2, b=3),
groups=set([TrialGroup(name='TG', project_id='1')]),
trials=set())
ps = from_json(to_json(p))
assert p == ps
def new_trial_group(self, group: TrialGroup):
try:
self.groups.insert_one(
to_json(group)
)
return group
except DuplicateKeyError:
return self.get_trial_group(group)
def get_project(self, project: Project):
kwargs = dict()
kwargs['__rpc__'] = 'get_project'
kwargs['project'] = to_json(project)
info(kwargs)
send(self.socket, kwargs)
p = _check(recv(self.socket))
info(f'got reply {p}')
return p
def exec(self, reader, writer, proc_name, proc, args, cache=None):
try:
new_args = self.process_args(args)
answer = proc(**new_args)
write(writer, {'status': 0, 'return': to_json(answer)})
# info(f'returned: {answer}')
except Exception as e:
error(f'An exception occurred while processing (rpc: {proc_name}) '
f'for user {self.get_username(reader)[0]}')
error(traceback.format_exc())
write(writer, {'status': 1, 'error': str(e)})
def create_experiment(self, config):
"""Insert a new experiment inside the database"""
self._get_project(config['name'])
self.group = self.backend.new_trial_group(
TrialGroup(name=experiment_uid(name=config['name'],
version=config['version']),
project_id=self.project.uid,
metadata=to_json(config)))
if self.group is None:
raise DuplicateKeyError('Experiment was already created')
config['_id'] = self.group.uid
return config
def test_trial():
p = Trial(name='name',
description='2',
tags=['0', '1'],
version='version',
group_id='0',
project_id='1',
parameters=dict(a=1, b=2),
metadata=dict(a=2, b=3),
metrics=dict(a=3, b=4),
chronos=dict(a=4, b=5),
status=Status.FinishedGroup,
errors=[])
ps = from_json(to_json(p))
# status is not serializable perfectly because we allow for custom status
ps.status = p.status
assert p == ps
def commit(self, file_name_override=None, **kwargs):
if file_name_override is None:
file_name_override = self.target_file
if file_name_override is None:
debug('No output file target')
return None
# only save top level projects
objects = []
for uid in self._projects:
objects.append(to_json(self._objects[uid]))
# print(json.dumps(objects[-1], indent=2))
file_name = tempfile.mktemp('track_uncommitted')
with open(file_name, 'w') as output:
json.dump(objects, output, indent=2)
# mv is kind of atomic so this prevent generating half generated files
os.rename(file_name, file_name_override)
def update_trial(self, trial=None, uid=None, **kwargs):
"""Update the fields of a given trials
Parameters
----------
trial: Trial
Trial object to update
where: Optional[dict]
constraint trial must respect
kwargs: dict
a dictionary of fields to update
Returns
-------
returns 1 if the underlying storage was updated else 0
"""
# Get a TrialAdapter
trial = self.get_trial(trial=trial, uid=uid)
try:
if isinstance(trial, TrialAdapter):
trial = trial.storage
for key, value in kwargs.items():
if key == "status":
self.backend.set_trial_status(trial,
get_track_status(value))
elif key in self._ignore_updates_for:
continue
else:
pair = {key: to_json(value)}
self.backend.log_trial_metadata(trial, **pair)
return 1
except ConcurrentWrite:
return 0
def _fetch_trials(self, query, *args, **kwargs):
"""Fetch all the trials that match the query"""
def sort_key(item):
submit_time = item.submit_time
if submit_time is None:
return 0
return submit_time
query = to_json(query)
new_query = {}
for k, v in query.items():
if k == "experiment":
new_query["group_id"] = v
elif k == "heartbeat":
new_query["metadata.heartbeat"] = v
elif k == "_id":
new_query["uid"] = v
elif k == "end_time":
new_query["metadata.end_time"] = v
elif k == "status" and isinstance(v, str):
new_query["status"] = get_track_status(v)
else:
new_query[k] = v
trials = [
TrialAdapter(t, objective=self.objective)
for t in self.backend.fetch_trials(new_query)
]
trials.sort(key=sort_key)
return trials
def new_trial_group(self, group: TrialGroup):
kwargs = dict()
kwargs['__rpc__'] = 'new_trial_group'
kwargs['group'] = to_json(group)
send(self.socket, kwargs)
return _check(recv(self.socket))
def fetch_trials(self, query):
query = {k: to_json(v) for k, v in query.items()}
return [from_json(g, dtype='trial') for g in self.trials.find(query)]
def serialize(obj):
return json.dumps(to_json(obj))
def report(self, short=True):
"""Print a digest of the logged metrics"""
self.logger.finish()
print(json.dumps(to_json(self.trial, short), indent=2))
return self
def set_trial_status(self, trial: Trial, status, error=None):
return self.trials.update_one(
{'uid': trial.uid},
{'$set': {
'status': to_json(status)}})
def new_trial(self, trial: Trial):
kwargs = dict()
kwargs['__rpc__'] = 'new_trial'
kwargs['trial'] = to_json(trial)
send(self.socket, kwargs)
return _check(recv(self.socket))
