def fetch_trials(self, query, selection=None):
"""Fetch trials of the experiment in the database
Trials are sorted based on `Trial.submit_time`
.. note::
The query is always updated with `{"experiment": self._id}`
.. seealso::
:meth:`orion.core.io.database.AbstractDB.read` for more information about the
arguments.
"""
query["experiment"] = self._id
trials = Trial.build(self._db.read('trials', query, selection))
def _get_submit_time(trial):
if trial.submit_time:
return trial.submit_time
return datetime.datetime.utcnow()
return list(sorted(trials, key=_get_submit_time))
def test_value_equal(self, exp_config):
"""Compare Param objects using __eq__"""
trials = Trial.build(exp_config[1])
assert trials[0].params[0] == Trial.Param(
**exp_config[1][0]['params'][0])
assert trials[0].params[1] != Trial.Param(
**exp_config[1][0]['params'][0])
def reserve_trial(self, score_handle=None):
"""Find *new* trials that exist currently in database and select one of
them based on the highest score return from `score_handle` callable.
:param score_handle: A way to decide which trial out of the *new* ones to
to pick as *reserved*, defaults to a random choice.
:type score_handle: callable
:return: selected `Trial` object, None if could not find any.
"""
if score_handle is not None and not callable(score_handle):
raise ValueError(
"Argument `score_handle` must be callable with a `Trial`.")
query = dict(experiment=self._id,
status={'$in': ['new', 'suspended', 'interrupted']})
new_trials = Trial.build(self._db.read('trials', query))
if not new_trials:
return None
if score_handle is not None and self.space:
scores = list(
map(score_handle,
map(lambda x: trial_to_tuple(x, self.space), new_trials)))
scored_trials = zip(scores, new_trials)
best_trials = filter(lambda st: st[0] == max(scores),
scored_trials)
new_trials = list(zip(*best_trials))[1]
elif score_handle is not None:
log.warning(
"While reserving trial: `score_handle` was provided, but "
"parameter space has not been defined yet.")
selected_trial = random.sample(new_trials, 1)[0]
# Query on status to ensure atomicity. If another process change the
# status meanwhile, read_and_write will fail, because query will fail.
query = {'_id': selected_trial.id, 'status': selected_trial.status}
update = dict(status='reserved')
if selected_trial.status == 'new':
update["start_time"] = datetime.datetime.utcnow()
selected_trial_dict = self._db.read_and_write('trials',
query=query,
data=update)
if selected_trial_dict is None:
selected_trial = self.reserve_trial(score_handle=score_handle)
else:
selected_trial = Trial(**selected_trial_dict)
return selected_trial
def _fetch_trials(self, query, selection=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_trials`"""
def sort_key(item):
submit_time = item.submit_time
if submit_time is None:
return 0
return submit_time
trials = Trial.build(
self._db.read("trials", query=query, selection=selection))
trials.sort(key=sort_key)
return trials
def fetch_completed_trials(self):
"""Fetch recent completed trials that this `Experiment` instance has not
yet seen.
.. note:: It will return only those with `Trial.end_time` after
`_last_fetched`, for performance reasons.
:return: list of completed `Trial` objects
"""
query = dict(experiment=self._id,
status='completed',
end_time={'$gte': self._last_fetched})
completed_trials = Trial.build(self._db.read('trials', query))
self._last_fetched = datetime.datetime.utcnow()
return completed_trials
def fetch_trials(self, query, selection=None):
"""Fetch trials of the experiment in the database
.. note::
The query is always updated with `{"experiment": self._id}`
.. seealso::
:meth:`orion.core.io.database.AbstractDB.read` for more information about the
arguments.
"""
query["experiment"] = self._id
return Trial.build(self._db.read('trials', query, selection))
def test_build_trials(self, exp_config):
"""Convert to objects form using `Trial.build`."""
trials = Trial.build(exp_config[1])
assert list(map(lambda x: x.to_dict(), trials)) == exp_config[1]
