def test_code_change_break_backward(create_db_instance):
"""Test that no trials pass to parent when code change type is 'break'"""
experiment_name = 'supernaedo2.3.1'
root_name = 'supernaedo2.3.1'
leaf_names = ['supernaedo2.3.1.3']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
children_trials = exp_node.children[0].item.fetch_trials(query)
assert len(children_trials) == 1
assert len(exp_node.item.fetch_trials(query)) == 2
adapter = exp_node.children[0].item.refers['adapter']
assert adapter.adapters[0].change_type == CodeChange.BREAK
adapted_children_trials = adapter.backward(children_trials)
assert len(adapted_children_trials) == 0
assert len(experiment.fetch_trials_tree(query)) == 2 + 0
def test_experiment_view_not_modified(self, exp_config, monkeypatch):
"""Experiment should not be modified if fetched in another verion of Oríon.
When loading a view the original config is used to configure the experiment, but
this process may modify the config if the version of Oríon is different. This should not be
saved in database.
"""
terrible_message = 'oh no, I have been modified!'
original_configuration = ExperimentView('supernaedo2').configuration
def modified_configuration(self):
mocked_config = copy.deepcopy(original_configuration)
mocked_config['metadata']['datetime'] = terrible_message
return mocked_config
with monkeypatch.context() as m:
m.setattr(Experiment, 'configuration',
property(modified_configuration))
exp = ExperimentView('supernaedo2')
# The mock is still in place and overwrites the configuration
assert exp.configuration['metadata'][
'datetime'] == terrible_message
# The mock is reverted and original config is returned, but modification is still in
# metadata
assert exp.metadata['datetime'] == terrible_message
# Loading again from DB confirms the DB was not overwritten
reloaded_exp = ExperimentView('supernaedo2')
assert reloaded_exp.configuration['metadata'][
'datetime'] != terrible_message
def test_full_forward_full_backward(create_db_instance):
"""Test that trials are adapted properly forward from parent and backward from leafs"""
experiment_name = 'supernaedo2.3'
root_name = 'supernaedo2'
leaf_names = []
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.name == root_name
assert exp_node.children[0].item.name == 'supernaedo2.3.1'
assert exp_node.children[0].children[0].item.name == 'supernaedo2.3.1.1'
assert exp_node.children[0].children[1].item.name == 'supernaedo2.3.1.2'
assert exp_node.children[0].children[2].item.name == 'supernaedo2.3.1.3'
# 2
# |
# 2.3
# |
# 2.3.1
# | \ \
# 2.3.1.1 2.3.1.2 2.3.1.3
assert len(list(exp_node.root)) == 6
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
assert len(exp_node.parent.item.fetch_trials(query)) == 6
assert len(exp_node.item.fetch_trials(query)) == 4
assert len(exp_node.children[0].item.fetch_trials(query)) == 2
assert len(exp_node.children[0].children[0].item.fetch_trials(query)) == 1
assert len(exp_node.children[0].children[1].item.fetch_trials(query)) == 1
assert len(exp_node.children[0].children[2].item.fetch_trials(query)) == 1
assert len(experiment.fetch_trials_tree(query)) == 6 + 4 + 1 + 1 + 0 + 0
def test_parent_parent_fetch_trials(create_db_instance):
"""Test that experiment fetch trials from grand parent properly (adapters are muted)"""
experiment_name = 'supernaedo2.3.1'
root_name = 'supernaedo2'
leaf_names = ['supernaedo2.3.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.parent.item.name == root_name
assert len(exp_node.children) == 0
# 2
# |
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 3
experiment.connect_to_version_control_tree(exp_node)
for node in exp_node.root:
node.item._experiment.refers['adapter'] = Adapter.build([])
query = {'status': 'completed'}
assert len(exp_node.parent.parent.item.fetch_trials(query)) == 6
assert len(exp_node.parent.item.fetch_trials(query)) == 4
assert len(exp_node.item.fetch_trials(query)) == 2
assert len(experiment.fetch_trials_tree(query)) == 6 + 4 + 2
def test_existing_experiment_view(self, create_db_instance, exp_config):
"""Hit exp_name + user's name in the db, fetch most recent entry."""
exp = ExperimentView('supernaedo2')
assert exp._experiment._init_done is False
assert exp._id == exp_config[0][0]['_id']
assert exp.name == exp_config[0][0]['name']
assert exp.configuration['refers'] == exp_config[0][0]['refers']
assert exp.metadata == exp_config[0][0]['metadata']
assert exp.pool_size == exp_config[0][0]['pool_size']
assert exp.max_trials == exp_config[0][0]['max_trials']
# TODO: Views are not fully configured until configuration is refactored
# assert exp.algorithms.configuration == exp_config[0][0]['algorithms']
with pytest.raises(AttributeError):
exp.this_is_not_in_config = 5
# Test that experiment.update_completed_trial indeed exists
exp._experiment.update_completed_trial
with pytest.raises(AttributeError):
exp.update_completed_trial
with pytest.raises(AttributeError):
exp.register_trial
with pytest.raises(AttributeError):
exp.reserve_trial
def test_full_forward(create_db_instance):
"""Test that trials are adapted properly down to leaf"""
experiment_name = 'supernaedo2.3.1.1'
root_name = None
leaf_names = []
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == 'supernaedo2.3.1'
assert exp_node.parent.parent.item.name == 'supernaedo2.3'
assert exp_node.parent.parent.parent.item.name == 'supernaedo2'
# 2
# |
# 2.3
# |
# 2.3.1
# |
# 2.3.1.1
assert len(list(exp_node.root)) == 4
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
assert len(exp_node.item.fetch_trials(query)) == 1
assert len(exp_node.parent.item.fetch_trials(query)) == 2
assert len(exp_node.parent.parent.item.fetch_trials(query)) == 4
assert len(exp_node.parent.parent.parent.item.fetch_trials(query)) == 6
assert len(experiment.fetch_trials_tree(query)) == 2 + 1 + 2 + 1
def test_code_change_unsure_forward(create_db_instance):
"""Test that all trials pass to children when code change type is 'unsure'"""
experiment_name = 'supernaedo2.3.1.2'
root_name = 'supernaedo2.3.1'
leaf_names = ['supernaedo2.3.1.2']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == root_name
# 2.3.1
# |
# 2.3.1.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
parent_trials = exp_node.parent.item.fetch_trials(query)
assert len(parent_trials) == 2
assert len(exp_node.item.fetch_trials(query)) == 1
adapter = experiment.refers['adapter']
assert adapter.adapters[0].change_type == CodeChange.UNSURE
adapted_parent_trials = adapter.forward(parent_trials)
assert len(adapted_parent_trials) == 2
assert len(experiment.fetch_trials_tree(query)) == 2 + 1
def test_existing_experiment_view(self, create_db_instance, exp_config):
"""Hit exp_name + user's name in the db, fetch most recent entry."""
exp = ExperimentView('supernaedo2')
assert exp._experiment._init_done is True
assert exp._experiment._db._database is create_db_instance
assert exp._id == exp_config[0][0]['_id']
assert exp.name == exp_config[0][0]['name']
assert exp.configuration['refers'] == exp_config[0][0]['refers']
assert exp.metadata == exp_config[0][0]['metadata']
assert exp._experiment._last_fetched == exp_config[0][0]['metadata'][
'datetime']
assert exp.pool_size == exp_config[0][0]['pool_size']
assert exp.max_trials == exp_config[0][0]['max_trials']
assert exp.algorithms.configuration == exp_config[0][0]['algorithms']
with pytest.raises(AttributeError):
exp.this_is_not_in_config = 5
# Test that experiment.push_completed_trial indeed exists
exp._experiment.push_completed_trial
with pytest.raises(AttributeError):
exp.push_completed_trial
with pytest.raises(AttributeError):
exp.register_trial
with pytest.raises(AttributeError):
exp.reserve_trial
def test_prior_change_forward(create_db_instance):
"""Test that trials from parent only pass to children if valid in the new prior"""
experiment_name = 'supernaedo2.3.1'
root_name = 'supernaedo2.3'
leaf_names = ['supernaedo2.3.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == root_name
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
parent_trials = exp_node.parent.item.fetch_trials(query)
assert len(parent_trials) == 4
assert len(exp_node.item.fetch_trials(query)) == 2
adapter = experiment.refers['adapter']
adapted_parent_trials = adapter.forward(parent_trials)
assert len(adapted_parent_trials) == 1
assert len(experiment.fetch_trials_tree(query)) == 2 + 1
def test_prior_change_backward(create_db_instance):
"""Test that all encoding are renamed to encoding_layer in parent"""
experiment_name = 'supernaedo2.3'
root_name = 'supernaedo2.3'
leaf_names = ['supernaedo2.3.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
children_trials = exp_node.children[0].item.fetch_trials(query)
assert len(children_trials) == 2
assert len(exp_node.item.fetch_trials(query)) == 4
adapter = exp_node.children[0].item.refers['adapter']
adapted_children_trials = adapter.backward(children_trials)
assert len(adapted_children_trials) == 1
assert len(experiment.fetch_trials_tree(query)) == 4 + 1
def test_renaming_forward(create_db_instance):
"""Test that all encoding_layer are renamed to encoding in children"""
experiment_name = 'supernaedo2.3'
root_name = 'supernaedo2'
leaf_names = ['supernaedo2.3']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == root_name
# 2
# |
# 2.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
parent_trials = exp_node.parent.item.fetch_trials(query)
assert len(parent_trials) == 6
assert len(exp_node.item.fetch_trials(query)) == 4
assert all((trial._params[0].name == "/encoding_layer") for trial in parent_trials)
adapter = experiment.refers['adapter']
adapted_parent_trials = adapter.forward(parent_trials)
assert len(adapted_parent_trials) == 6
assert all((trial._params[0].name == "/encoding") for trial in adapted_parent_trials)
assert len(experiment.fetch_trials_tree(query)) == 6 + 4
def test_deletion_adapter_backward(create_db_instance):
"""Test that all decoding_layer are passed with gru to parent"""
experiment_name = 'supernaedo2'
root_name = 'supernaedo2'
leaf_names = ['supernaedo2.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2
# |
# 2.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
assert len(exp_node.item.fetch_trials(query)) == 6
assert len(exp_node.children[0].item.fetch_trials(query)) == 1
adapter = exp_node.children[0].item.refers['adapter']
assert len(adapter.backward(exp_node.children[0].item.fetch_trials(query))) == 1
assert len(experiment.fetch_trials_tree(query)) == 6 + 1
def test_algo_change_backward(create_db_instance):
"""Test that all trials pass to parent when algorithm is changed"""
experiment_name = 'supernaedo2.2'
root_name = 'supernaedo2.2'
leaf_names = ['supernaedo2.2.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2.2
# |
# 2.2.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
children_trials = exp_node.children[0].item.fetch_trials(query)
assert len(children_trials) == 1
assert len(exp_node.item.fetch_trials(query)) == 1
adapter = exp_node.children[0].item.refers['adapter']
adapted_children_trials = adapter.backward(children_trials)
assert len(adapted_children_trials) == 1
assert len(experiment.fetch_trials_tree(query)) == 1 + 1
def test_fetch_completed_trials_from_view(hacked_exp, exp_config, random_dt):
"""Fetch a list of the unseen yet completed trials."""
experiment_view = ExperimentView(hacked_exp.name)
experiment_view._experiment = hacked_exp
trials = experiment_view.fetch_completed_trials()
assert len(trials) == 3
assert trials[0].to_dict() == exp_config[1][0]
assert trials[1].to_dict() == exp_config[1][2]
assert trials[2].to_dict() == exp_config[1][1]
def item(self):
"""Get the experiment associated to the node
Note that accessing `item` may trigger the lazy initialization of the experiment if it was
not done already.
"""
if self._item is None:
self._item = ExperimentView(self.name, version=self.version)
self._item.connect_to_version_control_tree(self)
return self._item
def test_fetch_completed_trials_from_view():
"""Fetch a list of the unseen yet completed trials."""
non_completed_stati = ['new', 'interrupted', 'suspended', 'reserved']
stati = non_completed_stati + ['completed']
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp_view = ExperimentView(exp)
trials = exp_view.fetch_trials_by_status('completed')
assert len(trials) == 1
assert trials[0].status == 'completed'
def test_experiment_view_stats(hacked_exp, exp_config, random_dt):
"""Check that property stats from view is consistent."""
experiment_view = ExperimentView(hacked_exp.name)
experiment_view._experiment = hacked_exp
stats = experiment_view.stats
assert stats['trials_completed'] == 3
assert stats['best_trials_id'] == exp_config[1][1]['_id']
assert stats['best_evaluation'] == 2
assert stats['start_time'] == exp_config[0][3]['metadata']['datetime']
assert stats['finish_time'] == exp_config[1][2]['end_time']
assert stats['duration'] == stats['finish_time'] - stats['start_time']
assert len(stats) == 6
def test_view_algo_is_done_property(hacked_exp):
"""Check experiment's algo stopping conditions accessed from view."""
experiment_view = ExperimentView(hacked_exp.name)
experiment_view._experiment = hacked_exp
# Fully configure wrapper experiment (should normally occur inside ExperimentView.__init__
# but hacked_exp has been _hacked_ inside afterwards.
hacked_exp.configure(hacked_exp.configuration)
assert experiment_view.is_done is False
hacked_exp.algorithms.algorithm.done = True
assert experiment_view.is_done is True
def test_empty_experiment_view_due_to_username(self):
"""Hit exp_name, but user's name does not hit the db, create new entry."""
with pytest.raises(ValueError) as exc_info:
ExperimentView('supernaedo2')
assert (
"No experiment with given name 'supernaedo2' for user 'bouthilx'"
in str(exc_info.value))
def build_view(name, version=None):
"""Build experiment view
An experiment view provides all reading operations of standard experiment but prevents the
modification of the experiment and its trials.
Parameters
----------
name: str
Name of the experiment to build
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
"""
db_config = fetch_config_from_db(name, version)
if not db_config:
message = (
"No experiment with given name '%s' and version '%s' inside database, "
"no view can be created." % (name, version if version else '*'))
raise ValueError(message)
db_config.setdefault('version', 1)
experiment = create_experiment(**db_config)
return ExperimentView(experiment)
def build_view_from(self, cmdargs):
"""Build an experiment view based on full configuration.
.. seealso::
`orion.core.io.experiment_builder` for more information on the hierarchy of
configurations.
:class:`orion.core.worker.experiment.ExperimentView` for more information on the
experiment view object.
"""
local_config = self.fetch_full_config(cmdargs, use_db=False)
self.setup_database(local_config)
# Information should be enough to infer experiment's name.
exp_name = local_config['name']
if exp_name is None:
raise RuntimeError(
"Could not infer experiment's name. "
"Please use either `name` cmd line arg or provide "
"one in orion's configuration file.")
return ExperimentView(local_config["name"],
local_config.get('user', None))
def build_view_from(self, cmdargs):
"""Build an experiment view based on full configuration.
.. seealso::
`orion.core.io.experiment_builder` for more information on the hierarchy of
configurations.
:class:`orion.core.worker.experiment.ExperimentView` for more information on the
experiment view object.
"""
local_config = self.fetch_full_config(cmdargs, use_db=False)
db_opts = local_config['database']
dbtype = db_opts.pop('type')
if local_config.get("debug"):
dbtype = "EphemeralDB"
# Information should be enough to infer experiment's name.
log.debug("Creating %s database client with args: %s", dbtype, db_opts)
try:
Database(of_type=dbtype, **db_opts)
except ValueError:
if Database().__class__.__name__.lower() != dbtype.lower():
raise
exp_name = local_config['name']
if exp_name is None:
raise RuntimeError("Could not infer experiment's name. "
"Please use either `name` cmd line arg or provide "
"one in orion's configuration file.")
return ExperimentView(local_config["name"], local_config.get('user', None))
def test_experiment_view_protocol_read_only():
"""Verify that wrapper experiments' database is read-only"""
exp = ExperimentView('supernaedo2')
# Test that _protocol.update_trials indeed exists
exp._experiment._storage._storage.update_trial
with pytest.raises(AttributeError):
exp._experiment._storage.update_trial
def test_experiment_view_db_read_only():
"""Verify that wrapper experiments' database is read-only"""
exp = ExperimentView('supernaedo2')
# Test that database.write indeed exists
exp._experiment._db._database.write
with pytest.raises(AttributeError):
exp._experiment._db.write
def test_view_is_done_property(hacked_exp):
"""Check experiment stopping conditions accessed from view."""
experiment_view = ExperimentView(hacked_exp.name)
experiment_view._experiment = hacked_exp
# Fully configure wrapper experiment (should normally occur inside ExperimentView.__init__
# but hacked_exp has been _hacked_ inside afterwards.
hacked_exp.configure(hacked_exp.configuration)
assert experiment_view.is_done is False
with pytest.raises(AttributeError):
experiment_view.max_trials = 2
hacked_exp.max_trials = 2
assert experiment_view.is_done is True
def test_experiment_view_protocol_read_only():
"""Verify that wrapper experiments' protocol is read-only"""
with OrionState():
exp = Experiment('supernaekei')
exp_view = ExperimentView(exp)
# Test that _protocol.set_trial_status indeed exists
exp_view._experiment._storage._storage.set_trial_status
with pytest.raises(AttributeError):
exp_view._experiment._storage.set_trial_status
def test_experiment_view_stats():
"""Check that property stats from view is consistent."""
NUM_COMPLETED = 3
stati = (['completed'] * NUM_COMPLETED) + (['reserved'] * 2)
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.metadata = {'datetime': datetime.datetime.utcnow()}
exp_view = ExperimentView(exp)
stats = exp_view.stats
assert stats['trials_completed'] == NUM_COMPLETED
assert stats['best_trials_id'] == cfg.trials[3]['_id']
assert stats['best_evaluation'] == 0
assert stats['start_time'] == exp_view.metadata['datetime']
assert stats['finish_time'] == cfg.trials[0]['end_time']
assert stats['duration'] == stats['finish_time'] - stats['start_time']
assert len(stats) == 6
def test_view_is_done_property_no_pending(algorithm):
"""Check experiment stopping conditions from view when there is no pending trials."""
completed = ['completed'] * 10
broken = ['broken'] * 5
with OrionState(trials=generate_trials(completed + broken)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.algorithms = algorithm
exp.max_trials = 100
exp_view = ExperimentView(exp)
exp.algorithms = algorithm
exp.max_trials = 15
# There is only 10 completed trials and algo not done.
assert not exp_view.is_done
exp.algorithms.algorithm.done = True
# Algorithm is done and no pending trials
assert exp_view.is_done
def test_view_is_done_property_with_pending(algorithm):
"""Check experiment stopping conditions from view when there is pending trials."""
completed = ['completed'] * 10
reserved = ['reserved'] * 5
with OrionState(trials=generate_trials(completed + reserved)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.algorithms = algorithm
exp.max_trials = 10
exp_view = ExperimentView(exp)
assert exp_view.is_done
exp.max_trials = 15
# There is only 10 completed trials
assert not exp_view.is_done
exp.algorithms.algorithm.done = True
# Algorithm is done but 5 trials are pending
assert not exp_view.is_done
def test_init(init_full_x, create_db_instance):
"""Test if original experiment contains trial 0"""
experiment = ExperimentView('full_x')
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 0), ), )
