def reserve_trial(self, score_handle=None):
"""Find *new* trials that exist currently in database and select one of
them based on the highest score return from `score_handle` callable.
:param score_handle: A way to decide which trial out of the *new* ones to
to pick as *reserved*, defaults to a random choice.
:type score_handle: callable
:return: selected `Trial` object, None if could not find any.
"""
if score_handle is not None and not callable(score_handle):
raise ValueError("Argument `score_handle` must be callable with a `Trial`.")
query = dict(
experiment=self._id,
status={'$in': ['new', 'suspended', 'interrupted']}
)
new_trials = Trial.build(self._db.read('trials', query))
if not new_trials:
return None
if score_handle is not None and self.space:
scores = list(map(score_handle,
map(lambda x: trial_to_tuple(x, self.space), new_trials)))
scored_trials = zip(scores, new_trials)
best_trials = filter(lambda st: st[0] == max(scores), scored_trials)
new_trials = list(zip(*best_trials))[1]
elif score_handle is not None:
log.warning("While reserving trial: `score_handle` was provided, but "
"parameter space has not been defined yet.")
selected_trial = random.sample(new_trials, 1)[0]
# Query on status to ensure atomicity. If another process change the
# status meanwhile, read_and_write will fail, because query will fail.
query = {'_id': selected_trial.id, 'status': selected_trial.status}
update = dict(status='reserved')
if selected_trial.status == 'new':
update["start_time"] = datetime.datetime.utcnow()
selected_trial_dict = self._db.read_and_write(
'trials', query=query, data=update)
if selected_trial_dict is None:
selected_trial = self.reserve_trial(score_handle=score_handle)
else:
selected_trial = Trial(**selected_trial_dict)
return selected_trial
def fetch_completed_trials(self):
"""Fetch recent completed trials that this `Experiment` instance has not
yet seen.
.. note:: It will return only those with `Trial.end_time` after
`_last_fetched`, for performance reasons.
:return: list of completed `Trial` objects
"""
query = dict(experiment=self._id,
status='completed',
end_time={'$gte': self._last_fetched})
completed_trials = Trial.build(self._db.read('trials', query))
self._last_fetched = datetime.datetime.utcnow()
return completed_trials
def reserve_trial(self, score_handle=None):
"""Find *new* trials that exist currently in database and select one of
them based on the highest score return from `score_handle` callable.
:param score_handle: A way to decide which trial out of the *new* ones to
to pick as *reserved*, defaults to a random choice.
:type score_handle: callable
:return: selected `Trial` object, None if could not find any.
"""
if score_handle is not None and not callable(score_handle):
raise ValueError(
"Argument `score_handle` must be callable with a `Trial`.")
query = dict(experiment=self._id,
status={'$in': ['new', 'suspended', 'interrupted']})
new_trials = Trial.build(self._db.read('trials', query))
if not new_trials:
return None
if score_handle is None:
selected_trial = random.sample(new_trials, 1)[0]
else:
raise NotImplementedError(
"scoring will be supported in the next iteration.")
if selected_trial.status == 'new':
selected_trial.start_time = datetime.datetime.utcnow()
selected_trial.status = 'reserved'
self._db.write('trials',
selected_trial.to_dict(),
query={'_id': selected_trial.id})
return selected_trial
def test_build_trials(self, exp_config):
"""Convert to objects form using `Trial.build`."""
trials = Trial.build(exp_config[1])
assert list(map(lambda x: x.to_dict(), trials)) == exp_config[1]