def test_build_no_hit(config_file, random_dt, script_path):
"""Try building experiment from config when not in db"""
name = 'supernaekei'
space = {'x': 'uniform(0, 10)'}
max_trials = 100
with OrionState(experiments=[], trials=[]):
with pytest.raises(ValueError) as exc_info:
experiment_builder.build_view(name)
assert "No experiment with given name 'supernaekei' and version '*'" in str(
exc_info.value)
exp = experiment_builder.build(name,
space=space,
max_trials=max_trials)
assert exp.name == name
assert exp.configuration['refers'] == {
'adapter': [],
'parent_id': None,
'root_id': exp._id
}
assert exp.metadata == {
'datetime': random_dt,
'user': '******',
'orion_version': 'XYZ'
}
assert exp.configuration['space'] == space
assert exp.max_trials == max_trials
assert not exp.is_done
assert exp.algorithms.configuration == {'random': {'seed': None}}
def test_empty_experiment_view(self):
"""Hit user name, but exp_name does not hit the db."""
with OrionState(experiments=[], trials=[]):
with pytest.raises(ValueError) as exc_info:
experiment_builder.build_view('supernaekei')
assert (
"No experiment with given name 'supernaekei' and version '*'"
in str(exc_info.value))
def get_experiments(args):
"""Return the different experiments.
Parameters
----------
args: dict
Commandline arguments.
"""
projection = {'name': 1, 'version': 1, 'refers': 1}
query = {'name': args['name']} if args.get('name') else {}
experiments = get_storage().fetch_experiments(query, projection)
if args['name']:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp['refers'].get('root_id', exp['_id']) == exp['_id']
]
return [
experiment_builder.build_view(name=exp['name'],
version=exp.get('version', 1))
for exp in root_experiments
]
def print_evc(experiments,
version=None,
all_trials=False,
collapse=False,
expand_versions=False,
**kwargs):
"""Print each EVC tree
Parameters
----------
args: dict
Commandline arguments.
"""
for exp in experiments:
experiment = experiment_builder.build_view(exp.name, version)
if version is None:
expand_experiment = exp
else:
expand_experiment = experiment
expand = expand_versions or _has_named_children(expand_experiment)
if expand and not collapse:
print_status_recursively(expand_experiment, all_trials=all_trials)
else:
print_status(experiment, all_trials=all_trials, collapse=True)
def main(args):
"""List all experiments inside database."""
config = experiment_builder.get_cmd_config(args)
experiment_builder.setup_storage(config.get('storage'))
query = {}
if args['name']:
query['name'] = args['name']
experiments = get_storage().fetch_experiments(query)
if args['name']:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp['refers'].get('root_id', exp['_id']) == exp['_id']
]
if not root_experiments:
print("No experiment found")
return
for root_experiment in root_experiments:
root = experiment_builder.build_view(
name=root_experiment['name'],
version=root_experiment.get('version')).node
print_tree(root, nameattr='tree_name')
def test_existing_experiment_view(self, new_config):
"""Hit exp_name + user's name in the db, fetch most recent entry."""
with OrionState(experiments=[new_config], trials=[]):
exp = experiment_builder.build_view(name='supernaekei')
assert exp._id == new_config['_id']
assert exp.name == new_config['name']
assert exp.configuration['refers'] == new_config['refers']
assert exp.metadata == new_config['metadata']
assert exp.pool_size == new_config['pool_size']
assert exp.max_trials == new_config['max_trials']
assert exp.version == new_config['version']
assert isinstance(exp.refers['adapter'], BaseAdapter)
assert exp.algorithms.configuration == new_config['algorithms']
with pytest.raises(AttributeError):
exp.this_is_not_in_config = 5
# Test that experiment.update_completed_trial indeed exists
exp._experiment.update_completed_trial
with pytest.raises(AttributeError):
exp.update_completed_trial
with pytest.raises(AttributeError):
exp.register_trial
with pytest.raises(AttributeError):
exp.reserve_trial
def test_build_without_config_hit(python_api_config):
"""Try building experiment without commandline config when in db (no branch)"""
name = 'supernaekei'
with OrionState(experiments=[python_api_config], trials=[]):
# Test that experiment already exists (this should fail otherwise)
experiment_builder.build_view(name=name)
exp = experiment_builder.build(name=name)
assert exp._id == python_api_config['_id']
assert exp.name == python_api_config['name']
assert exp.configuration['refers'] == python_api_config['refers']
assert exp.metadata == python_api_config['metadata']
assert exp.max_trials == python_api_config['max_trials']
assert exp.algorithms.configuration == python_api_config['algorithms']
def test_full_x_remove_y(init_full_x_remove_y, create_db_instance):
"""Test if full x removed y is properly initialized and can fetch from its 2 parents"""
experiment = experiment_builder.build_view(name='full_x_remove_y')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 7), ), (('/x', -7), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ),
(('/x', 7), ), (('/x', -7), ))
def test_full_x_remove_z(init_full_x_remove_z, create_db_instance):
"""Test if full x removed z is properly initialized and can fetch from 2 of its 3 parents"""
experiment = experiment_builder.build_view(name='full_x_remove_z')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 8), ), (('/x', -8), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
# Note that full_x_rename_y_z are filtered out because default_value=1
assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ),
(('/x', 8), ), (('/x', -8), ))
def test_half_x_full_y(init_half_x_full_y, create_db_instance):
"""Test if half x full y is properly initialized and can fetch from its 2 parents"""
experiment = experiment_builder.build_view(name='half_x_full_y')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 2), ('/y', 2)), (('/x', 2), ('/y', -2)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', 2), ('/y', 2)),
(('/x', 2), ('/y', -2)))
def test_full_x_full_y(init_full_x_full_y, create_db_instance):
"""Test if full x full y is properly initialized and can fetch original trial"""
experiment = experiment_builder.build_view(name='full_x_full_y')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)), (('/x', 1), ('/y', -1)),
(('/x', -1), ('/y', -1)))
def test_new_cli(init_full_x_new_cli):
"""Test that new cli conflict is automatically resolved"""
experiment = experiment_builder.build_view(name='full_x_new_cli')
assert len(experiment.fetch_trials(with_evc_tree=True)) == 3
assert len(experiment.fetch_trials()) == 2
orion.core.cli.main((
"-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_cli").split(" "))
assert len(experiment.fetch_trials(with_evc_tree=True)) == 21
assert len(experiment.fetch_trials()) == 20
def test_run_entire_full_x_full_y_no_args(init_entire, create_db_instance):
"""Test if branched experiment can be executed without script arguments"""
experiment = experiment_builder.build_view(name='full_x_full_y')
assert len(experiment.fetch_trials(with_evc_tree=True)) == 23
assert len(experiment.fetch_trials()) == 4
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y").split(" "))
assert len(experiment.fetch_trials(with_evc_tree=True)) == 39
assert len(experiment.fetch_trials()) == 20
def item(self):
"""Get the experiment associated to the node
Note that accessing `item` may trigger the lazy initialization of the experiment if it was
not done already.
"""
if self._item is None:
# TODO: Find another way around the circular import
import orion.core.io.experiment_builder as experiment_builder
self._item = experiment_builder.build_view(name=self.name, version=self.version)
self._item._experiment._node = self
return self._item
def test_full_x_rename_half_y_half_z(init_full_x_rename_half_y_half_z,
create_db_instance):
"""Test if full x half z is properly initialized and can fetch from its 3 parents"""
experiment = experiment_builder.build_view(
name='full_x_rename_half_y_half_z')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 5), ('/z', 5)), (('/x', -5), ('/z', 5)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ('/z', 1)), (('/x', 1), ('/z', 1)),
(('/x', -1), ('/z', 1)), (('/x', 3), ('/z', 3)),
(('/x', -3), ('/z', 3)), (('/x', 5), ('/z', 5)),
(('/x', -5), ('/z', 5)))
def test_run_entire_full_x_full_y(init_entire, create_db_instance):
"""Test if branched experiment can be executed without triggering a branching event again"""
experiment = experiment_builder.build_view(name='full_x_full_y')
assert len(experiment.fetch_trials(with_evc_tree=True)) == 23
assert len(experiment.fetch_trials()) == 4
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y "
"./black_box_with_y.py "
"-x~uniform(-10,10) "
"-y~uniform(-10,10,default_value=1)").split(" "))
assert len(experiment.fetch_trials(with_evc_tree=True)) == 39
assert len(experiment.fetch_trials()) == 20
def test_full_x_remove_z_default_4(init_full_x_remove_z_default_4,
create_db_instance):
"""Test if full x removed z (default 4) is properly initialized and can fetch
from 1 of its 3 parents
"""
experiment = experiment_builder.build_view(
name='full_x_remove_z_default_4')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 9), ), (('/x', -9), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
# Note that full_x and full_x_full_y are filtered out because default_value=4
assert pairs == ((('/x', 4), ), (('/x', -4), ), (('/x', 9), ), (('/x',
-9), ))
def test_entire_full_x_full_y(init_entire, create_db_instance):
"""Test if full x full y can fetch from its parent and all children"""
experiment = experiment_builder.build_view(name='full_x_full_y')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert set(pairs) == set((
(('/x', 0), ('/y', 1)),
# full_x_full_y
(('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)),
(('/x', -1), ('/y', -1)),
# half_x_full_y
(('/x', 2), ('/y', 2)),
(('/x', 2), ('/y', -2)),
# full_x_half_y
(('/x', 3), ('/y', 3)),
(('/x', -3), ('/y', 3)),
# full_x_rename_y_z
(('/x', 4), ('/y', 4)),
(('/x', -4), ('/y', 4)),
(('/x', 4), ('/y', -4)),
(('/x', -4), ('/y', -4)),
# full_x_rename_half_y_half_z
(('/x', 5), ('/y', 5)),
(('/x', -5), ('/y', 5)),
# full_x_rename_half_y_full_z
(('/x', 6), ('/y', 6)),
(('/x', -6), ('/y', 6)),
# full_x_remove_y
(('/x', 7), ('/y', 1)),
(('/x', -7), ('/y', 1)),
# full_x_remove_z
(('/x', 8), ('/y', 1)),
(('/x', -8), ('/y', 1)),
# full_x_remove_z_default_4
(('/x', 9), ('/y', 4)),
(('/x', -9), ('/y', 4))))
def test_init(init_full_x, create_db_instance):
"""Test if original experiment contains trial 0"""
experiment = experiment_builder.build_view(name='full_x')
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 0), ), )
