def test_register_trials(database, random_dt, hacked_exp):
"""Register a list of newly proposed trials/parameters."""
hacked_exp._id = 'lalala' # white box hack
trials = [
Trial(params=[{
'name': 'a',
'type': 'int',
'value': 5
}]),
Trial(params=[{
'name': 'b',
'type': 'int',
'value': 6
}]),
]
hacked_exp.register_trials(trials)
yo = list(database.trials.find({'experiment': hacked_exp._id}))
assert len(yo) == len(trials)
assert yo[0]['params'] == list(map(lambda x: x.to_dict(),
trials[0].params))
assert yo[1]['params'] == list(map(lambda x: x.to_dict(),
trials[1].params))
assert yo[0]['status'] == 'new'
assert yo[1]['status'] == 'new'
assert yo[0]['submit_time'] == random_dt
assert yo[1]['submit_time'] == random_dt
def test_not_allowed_status(self):
"""Other than `Trial.allowed_stati` are not allowed in `Trial.status`."""
t = Trial()
with pytest.raises(ValueError):
t.status = 'asdf'
with pytest.raises(ValueError):
t = Trial(status='ispi')
def stats(self):
"""Calculate a stats dictionary for this particular experiment.
Returns
-------
stats : dict
Stats
-----
trials_completed : int
Number of completed trials
best_trials_id : int
Unique identifier of the `Trial` object in the database which achieved
the best known objective result.
best_evaluation : float
Evaluation score of the best trial
start_time : `datetime.datetime`
When Experiment was first dispatched and started running.
finish_time : `datetime.datetime`
When Experiment reached terminating condition and stopped running.
duration : `datetime.timedelta`
Elapsed time.
"""
query = dict(experiment=self._id, status='completed')
completed_trials = self._db.read('trials',
query,
selection={
'_id': 1,
'end_time': 1,
'results': 1
})
stats = dict()
stats['trials_completed'] = len(completed_trials)
stats['best_trials_id'] = None
trial = Trial(**completed_trials[0])
stats['best_evaluation'] = trial.objective.value
stats['best_trials_id'] = trial.id
stats['start_time'] = self.metadata['datetime']
stats['finish_time'] = stats['start_time']
for trial in completed_trials:
trial = Trial(**trial)
# All trials are going to finish certainly after the start date
# of the experiment they belong to
if trial.end_time > stats['finish_time']: # pylint:disable=no-member
stats['finish_time'] = trial.end_time
objective = trial.objective.value
if objective < stats['best_evaluation']:
stats['best_evaluation'] = objective
stats['best_trials_id'] = trial.id
stats['duration'] = stats['finish_time'] - stats['start_time']
return stats
def reserve_trial(self, score_handle=None):
"""Find *new* trials that exist currently in database and select one of
them based on the highest score return from `score_handle` callable.
:param score_handle: A way to decide which trial out of the *new* ones to
to pick as *reserved*, defaults to a random choice.
:type score_handle: callable
:return: selected `Trial` object, None if could not find any.
"""
if score_handle is not None and not callable(score_handle):
raise ValueError("Argument `score_handle` must be callable with a `Trial`.")
query = dict(
experiment=self._id,
status={'$in': ['new', 'suspended', 'interrupted']}
)
new_trials = Trial.build(self._db.read('trials', query))
if not new_trials:
return None
if score_handle is not None and self.space:
scores = list(map(score_handle,
map(lambda x: trial_to_tuple(x, self.space), new_trials)))
scored_trials = zip(scores, new_trials)
best_trials = filter(lambda st: st[0] == max(scores), scored_trials)
new_trials = list(zip(*best_trials))[1]
elif score_handle is not None:
log.warning("While reserving trial: `score_handle` was provided, but "
"parameter space has not been defined yet.")
selected_trial = random.sample(new_trials, 1)[0]
# Query on status to ensure atomicity. If another process change the
# status meanwhile, read_and_write will fail, because query will fail.
query = {'_id': selected_trial.id, 'status': selected_trial.status}
update = dict(status='reserved')
if selected_trial.status == 'new':
update["start_time"] = datetime.datetime.utcnow()
selected_trial_dict = self._db.read_and_write(
'trials', query=query, data=update)
if selected_trial_dict is None:
selected_trial = self.reserve_trial(score_handle=score_handle)
else:
selected_trial = Trial(**selected_trial_dict)
return selected_trial
def test_value_not_allowed_type(self):
"""Other than `Trial.Result.allowed_types` are not allowed in `Trial.Result.type`.
Same for `Trial.Param`.
"""
with pytest.raises(ValueError):
v = Trial.Result(name='asfda', type='hoho')
v = Trial.Result()
with pytest.raises(ValueError):
v.type = 'asfda'
with pytest.raises(ValueError):
v = Trial.Param(name='asfda', type='hoho')
v = Trial.Param()
with pytest.raises(ValueError):
v.type = 'asfda'
def trial():
"""Stab trial to match tuple from fixture `fixed_suggestion`."""
params = [
dict(name='yolo', type='categorical', value=('asdfa', 2)),
dict(name='yolo2', type='integer', value=0),
dict(name='yolo3', type='real', value=3.5)
]
return Trial(params=params)
def test_generate_only_with_json_config(self, spacebuilder,
json_sample_path, tmpdir,
json_converter):
"""Build a space using only a json config."""
spacebuilder.build_from(['--config=' + json_sample_path])
trial = Trial(params=[{
'name': '/layers/1/width',
'type': 'integer',
'value': 100
}, {
'name': '/layers/1/type',
'type': 'categorical',
'value': 'relu'
}, {
'name': '/layers/2/type',
'type': 'categorical',
'value': 'sigmoid'
}, {
'name': '/training/lr0',
'type': 'real',
'value': 0.032
}, {
'name': '/training/mbs',
'type': 'integer',
'value': 64
}, {
'name': '/something-same',
'type': 'categorical',
'value': '3'
}])
output_file = str(tmpdir.join("output.json"))
cmd_inst = spacebuilder.build_to(output_file, trial)
assert cmd_inst == ['--config=' + output_file]
output_data = json_converter.parse(output_file)
assert output_data == {
'yo':
5,
'training': {
'lr0': 0.032,
'mbs': 64
},
'layers': [{
'width': 64,
'type': 'relu'
}, {
'width': 100,
'type': 'relu'
}, {
'width': 16,
'type': 'sigmoid'
}],
'something-same':
'3'
}
def test_init_empty(self):
"""Initialize empty trial."""
t = Trial()
assert t.experiment is None
assert t.status == 'new'
assert t.worker is None
assert t.submit_time is None
assert t.start_time is None
assert t.end_time is None
assert t.results == []
assert t.params == []
def test_push_completed_trial(hacked_exp, database, random_dt):
"""Successfully push a completed trial into database."""
trial = hacked_exp.reserve_trial()
trial.results = []
res = Trial.Result(name='yolo', type='objective', value='3')
trial.results.append(res)
hacked_exp.push_completed_trial(trial)
yo = database.trials.find_one({'_id': trial.id})
assert len(yo['results']) == len(trial.results)
assert yo['results'][0] == trial.results[0].to_dict()
assert yo['status'] == 'completed'
assert yo['end_time'] == random_dt
def tuple_to_trial(data, space):
"""Create a `metaopt.core.worker.trial.Trial` object from `data`,
filling only parameter information from `data`.
:param data: A tuple representing a sample point from `space`.
:param space: Definition of problem's domain.
:type space: `metaopt.algo.space.Space`
"""
assert len(data) == len(space)
params = [
dict(name=space[order].name, type=space[order].type, value=data[order])
for order in range(len(space))
]
return Trial(params=params)
def test_gradient_property(self, exp_config):
"""Check property `Trial.gradient`."""
# 1 results in `results` list
t = Trial(**exp_config[1][2])
assert isinstance(t.gradient, Trial.Result)
assert t.gradient.name == 'naedw_grad'
assert t.gradient.type == 'gradient'
assert t.gradient.value == [5, 3]
# 0 results in `results` list
tmp = exp_config[1][2]['results'].pop()
t = Trial(**exp_config[1][2])
assert t.gradient is None
exp_config[1][2]['results'].append(tmp)
# >1 results in `results` list
exp_config[1][2]['results'].append(
dict(name='yolo2', type='gradient', value=[12, 15]))
t = Trial(**exp_config[1][2])
assert isinstance(t.gradient, Trial.Result)
assert t.gradient.name == 'naedw_grad'
assert t.gradient.type == 'gradient'
assert t.gradient.value == [5, 3]
tmp = exp_config[1][2]['results'].pop()
def test_objective_property(self, exp_config):
"""Check property `Trial.objective`."""
# 1 results in `results` list
t = Trial(**exp_config[1][2])
assert isinstance(t.objective, Trial.Result)
assert t.objective.name == 'yolo'
assert t.objective.type == 'objective'
assert t.objective.value == 10
# 0 results in `results` list
tmp = exp_config[1][2]['results'].pop(0)
t = Trial(**exp_config[1][2])
assert t.objective is None
exp_config[1][2]['results'].append(tmp)
# >1 results in `results` list
exp_config[1][2]['results'].append(
dict(name='yolo2', type='objective', value=12))
t = Trial(**exp_config[1][2])
assert isinstance(t.objective, Trial.Result)
assert t.objective.name == 'yolo'
assert t.objective.type == 'objective'
assert t.objective.value == 10
tmp = exp_config[1][2]['results'].pop()
def test_init_full(self, exp_config):
"""Initialize with a dictionary with complete specification."""
t = Trial(**exp_config[1][1])
assert t.experiment == exp_config[1][1]['experiment']
assert t.status == exp_config[1][1]['status']
assert t.worker == exp_config[1][1]['worker']
assert t.submit_time == exp_config[1][1]['submit_time']
assert t.start_time == exp_config[1][1]['start_time']
assert t.end_time == exp_config[1][1]['end_time']
assert list(map(lambda x: x.to_dict(),
t.results)) == exp_config[1][1]['results']
assert t.results[0].name == exp_config[1][1]['results'][0]['name']
assert t.results[0].type == exp_config[1][1]['results'][0]['type']
assert t.results[0].value == exp_config[1][1]['results'][0]['value']
assert list(map(lambda x: x.to_dict(),
t.params)) == exp_config[1][1]['params']
def fetch_completed_trials(self):
"""Fetch recent completed trials that this `Experiment` instance has not
yet seen.
.. note:: It will return only those with `Trial.end_time` after
`_last_fetched`, for performance reasons.
:return: list of completed `Trial` objects
"""
query = dict(experiment=self._id,
status='completed',
end_time={'$gte': self._last_fetched})
completed_trials = Trial.build(self._db.read('trials', query))
self._last_fetched = datetime.datetime.utcnow()
return completed_trials
def _consume(self, trial, workdirname):
config_file = tempfile.NamedTemporaryFile(mode='w',
prefix='trial_',
suffix='.conf',
dir=workdirname,
delete=False)
config_file.close()
log.debug("## New temp config file: %s", config_file.name)
results_file = tempfile.NamedTemporaryFile(mode='w',
prefix='results_',
suffix='.log',
dir=workdirname,
delete=False)
results_file.close()
log.debug("## New temp results file: %s", results_file.name)
log.debug(
"## Building command line argument and configuration for trial.")
cmd_args = self.template_builder.build_to(config_file.name, trial)
log.debug(
"## Launch user's script as a subprocess and wait for finish.")
script_process = self.launch_process(results_file.name, cmd_args)
if script_process is None:
return None
returncode = script_process.wait()
if returncode != 0:
log.error(
"Something went wrong. Check logs. Process "
"returned with code %d !", returncode)
return None
log.debug(
"## Parse results from file and fill corresponding Trial object.")
results = self.converter.parse(results_file.name)
trial.results = [
Trial.Result(name=res['name'],
type=res['type'],
value=res['value']) for res in results
]
return trial
def test_generate_without_config(self, spacebuilder):
"""Build a space using only args."""
cmd_args = [
"--seed=555", "-yolo~uniform(-3, 1)",
"--arch1=choices({'lala': 0.2, 'yolo': 0.8})",
"--arch2~choices({'lala': 0.2, 'yolo': 0.8})"
]
spacebuilder.build_from(cmd_args)
trial = Trial(params=[{
'name': '/yolo',
'type': 'real',
'value': -2.4
}, {
'name': '/arch2',
'type': 'categorical',
'value': 'yolo'
}])
cmd_inst = spacebuilder.build_to(None, trial)
assert cmd_inst == [
"--seed=555", "--arch1=choices({'lala': 0.2, 'yolo': 0.8})",
"-yolo=-2.4", "--arch2=yolo"
]
def reserve_trial(self, score_handle=None):
"""Find *new* trials that exist currently in database and select one of
them based on the highest score return from `score_handle` callable.
:param score_handle: A way to decide which trial out of the *new* ones to
to pick as *reserved*, defaults to a random choice.
:type score_handle: callable
:return: selected `Trial` object, None if could not find any.
"""
if score_handle is not None and not callable(score_handle):
raise ValueError(
"Argument `score_handle` must be callable with a `Trial`.")
query = dict(experiment=self._id,
status={'$in': ['new', 'suspended', 'interrupted']})
new_trials = Trial.build(self._db.read('trials', query))
if not new_trials:
return None
if score_handle is None:
selected_trial = random.sample(new_trials, 1)[0]
else:
raise NotImplementedError(
"scoring will be supported in the next iteration.")
if selected_trial.status == 'new':
selected_trial.start_time = datetime.datetime.utcnow()
selected_trial.status = 'reserved'
self._db.write('trials',
selected_trial.to_dict(),
query={'_id': selected_trial.id})
return selected_trial
def test_bad_access(self):
"""Other than `Trial.__slots__` are not allowed."""
t = Trial()
with pytest.raises(AttributeError):
t.asdfa = 3
def test_generate_from_args_and_config(self, spacebuilder,
json_sample_path, tmpdir,
json_converter):
"""Build a space using only a json config."""
cmd_args = [
"--seed=555", "-yolo~uniform(-3, 1)",
'--config=' + json_sample_path,
"--arch1=choices({'lala': 0.2, 'yolo': 0.8})",
"--arch2~choices({'lala': 0.2, 'yolo': 0.8})"
]
spacebuilder.build_from(cmd_args)
trial = Trial(params=[{
'name': '/yolo',
'type': 'real',
'value': -2.4
}, {
'name': '/arch2',
'type': 'categorical',
'value': 'yolo'
}, {
'name': '/layers/1/width',
'type': 'integer',
'value': 100
}, {
'name': '/layers/1/type',
'type': 'categorical',
'value': 'relu'
}, {
'name': '/layers/2/type',
'type': 'categorical',
'value': 'sigmoid'
}, {
'name': '/training/lr0',
'type': 'real',
'value': 0.032
}, {
'name': '/training/mbs',
'type': 'integer',
'value': 64
}, {
'name': '/something-same',
'type': 'categorical',
'value': '3'
}])
output_file = str(tmpdir.join("output.json"))
cmd_inst = spacebuilder.build_to(output_file, trial)
assert cmd_inst == ['--config=' + output_file] + [
"--seed=555", "--arch1=choices({'lala': 0.2, 'yolo': 0.8})",
"-yolo=-2.4", "--arch2=yolo"
]
output_data = json_converter.parse(output_file)
assert output_data == {
'yo':
5,
'training': {
'lr0': 0.032,
'mbs': 64
},
'layers': [{
'width': 64,
'type': 'relu'
}, {
'width': 100,
'type': 'relu'
}, {
'width': 16,
'type': 'sigmoid'
}],
'something-same':
'3'
}
def test_value_bad_init(self):
"""Other than `Trial.Value.__slots__` are not allowed in __init__ too."""
with pytest.raises(AttributeError):
Trial.Value(ispii='iela')
def test_convertion_to_dict(self, exp_config):
"""Convert to dictionary form for database using ``dict``."""
t = Trial(**exp_config[1][1])
assert t.to_dict() == exp_config[1][1]
def test_build_trials(self, exp_config):
"""Convert to objects form using `Trial.build`."""
trials = Trial.build(exp_config[1])
assert list(map(lambda x: x.to_dict(), trials)) == exp_config[1]
def test_str_trial(self, exp_config):
"""Test representation of `Trial`."""
t = Trial(**exp_config[1][2])
assert str(t) == "Trial(experiment='supernaedo2', "\
"status='completed', params.value=['gru', 'lstm_with_attention'])"
def test_str_value(self, exp_config):
"""Test representation of `Trial.Value`."""
t = Trial(**exp_config[1][2])
assert str(t.params[0]) == "Param(name='/encoding_layer', "\
"type='categorical', value='gru')"
