def test_full_x_remove_y(init_full_x_remove_y, create_db_instance):
"""Test if full x removed y is properly initialized and can fetch from its 2 parents"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_remove_y'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 7), ), (('/x', -7), ))
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ),
(('/x', 7), ), (('/x', -7), ))
def test_full_x_remove_z(init_full_x_remove_z, create_db_instance):
"""Test if full x removed z is properly initialized and can fetch from 2 of its 3 parents"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_remove_z'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 8), ), (('/x', -8), ))
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
# Note that full_x_rename_y_z are filtered out because default_value=1
assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ),
(('/x', 8), ), (('/x', -8), ))
def test_full_x_half_y(init_full_x_half_y, create_db_instance):
"""Test if full x half y is properly initialized and can fetch from its 2 parents"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_half_y'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 3), ('/y', 3)), (('/x', -3), ('/y', 3)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)), (('/x', 3), ('/y', 3)),
(('/x', -3), ('/y', 3)))
def test_half_x_full_y(init_half_x_full_y, create_db_instance):
"""Test if half x full y is properly initialized and can fetch from its 2 parents"""
experiment = EVCBuilder().build_view_from({'name': 'half_x_full_y'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 2), ('/y', 2)), (('/x', 2), ('/y', -2)))
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', 2), ('/y', 2)),
(('/x', 2), ('/y', -2)))
def test_full_x_full_y(init_full_x_full_y, create_db_instance):
"""Test if full x full y is properly initialized and can fetch original trial"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)))
# pytest.set_trace()
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)), (('/x', 1), ('/y', -1)),
(('/x', -1), ('/y', -1)))
def test_full_x_rename_half_y_half_z(init_full_x_rename_half_y_half_z,
create_db_instance):
"""Test if full x half z is properly initialized and can fetch from its 3 parents"""
experiment = EVCBuilder().build_view_from(
{'name': 'full_x_rename_half_y_half_z'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 5), ('/z', 5)), (('/x', -5), ('/z', 5)))
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
assert pairs == ((('/x', 0), ('/z', 1)), (('/x', 1), ('/z', 1)),
(('/x', -1), ('/z', 1)), (('/x', 3), ('/z', 3)),
(('/x', -3), ('/z', 3)), (('/x', 5), ('/z', 5)),
(('/x', -5), ('/z', 5)))
def test_full_x_remove_z_default_4(init_full_x_remove_z_default_4,
create_db_instance):
"""Test if full x removed z (default 4) is properly initialized and can fetch
from 1 of its 3 parents
"""
experiment = EVCBuilder().build_view_from(
{'name': 'full_x_remove_z_default_4'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 9), ), (('/x', -9), ))
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
# Note that full_x and full_x_full_y are filtered out because default_value=4
assert pairs == ((('/x', 4), ), (('/x', -4), ), (('/x', 9), ), (('/x',
-9), ))
def main(args):
"""List all experiments inside database."""
builder = ExperimentBuilder()
config = builder.fetch_full_config(args, use_db=False)
builder.setup_storage(config)
query = {}
if args['name']:
query['name'] = args['name']
experiments = get_storage().fetch_experiments(query)
if args['name']:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp['refers'].get('root_id', exp['_id']) == exp['_id']
]
for root_experiment in root_experiments:
root = EVCBuilder().build_view_from({
'name':
root_experiment['name'],
'version':
root_experiment.get('version')
}).node
print_tree(root, nameattr='tree_name')
def main(args):
"""List all experiments inside database."""
builder = ExperimentBuilder()
config = builder.fetch_full_config(args, use_db=False)
builder.setup_database(config)
query = {}
if args['name']:
query['name'] = args['name']
experiments = Database().read("experiments", query)
if args['name']:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp['refers'].get('root_id', exp['_id']) == exp['_id']
]
for root_experiment in root_experiments:
root = EVCBuilder().build_view_from({
'name': root_experiment['name']
}).node
print_tree(root)
def print_evc(experiments,
version=None,
all_trials=False,
collapse=False,
expand_versions=False,
**kwargs):
"""Print each EVC tree
Parameters
----------
args: dict
Commandline arguments.
"""
for exp in experiments:
cfg = {'name': exp.name, 'version': version}
experiment = EVCBuilder().build_view_from(cfg)
if version is None:
expand_experiment = exp
else:
expand_experiment = experiment
expand = expand_versions or _has_named_children(expand_experiment)
if expand and not collapse:
print_status_recursively(expand_experiment, all_trials=all_trials)
else:
print_status(experiment, all_trials=all_trials, collapse=True)
def main(args):
"""Build experiment and execute hunt command"""
args['root'] = None
args['leafs'] = []
# TODO: simplify when parameter parsing is refactored
worker_trials = ExperimentBuilder().fetch_full_config(args)['worker_trials']
experiment = EVCBuilder().build_from(args)
workon(experiment, worker_trials)
def main(args):
"""Fetch config and info experiments"""
try:
experiment = EVCBuilder().build_view_from(args)
except ValueError:
print('Experiment {} not found in db.'.format(args.get('name', None)))
sys.exit(1)
hack_until_config_is_refactored(experiment)
print(format_info(experiment))
def test_entire_full_x_full_y(init_entire, create_db_instance):
"""Test if full x full y can fetch from its parent and all children"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)))
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
assert set(pairs) == set((
(('/x', 0), ('/y', 1)),
# full_x_full_y
(('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)),
(('/x', -1), ('/y', -1)),
# half_x_full_y
(('/x', 2), ('/y', 2)),
(('/x', 2), ('/y', -2)),
# full_x_half_y
(('/x', 3), ('/y', 3)),
(('/x', -3), ('/y', 3)),
# full_x_rename_y_z
(('/x', 4), ('/y', 4)),
(('/x', -4), ('/y', 4)),
(('/x', 4), ('/y', -4)),
(('/x', -4), ('/y', -4)),
# full_x_rename_half_y_half_z
(('/x', 5), ('/y', 5)),
(('/x', -5), ('/y', 5)),
# full_x_rename_half_y_full_z
(('/x', 6), ('/y', 6)),
(('/x', -6), ('/y', 6)),
# full_x_remove_y
(('/x', 7), ('/y', 1)),
(('/x', -7), ('/y', 1)),
# full_x_remove_z
(('/x', 8), ('/y', 1)),
(('/x', -8), ('/y', 1)),
# full_x_remove_z_default_4
(('/x', 9), ('/y', 4)),
(('/x', -9), ('/y', 4))))
def test_run_entire_full_x_full_y_no_args(init_entire, create_db_instance):
"""Test if branched experiment can be executed without script arguments"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
assert len(experiment.fetch_trials_tree({})) == 23
assert len(experiment.fetch_trials({})) == 4
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y").split(" "))
assert len(experiment.fetch_trials_tree({})) == 39
assert len(experiment.fetch_trials({})) == 20
def test_new_cli(init_full_x_new_cli):
"""Test that new cli conflict is automatically resolved"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_new_cli'})
assert len(experiment.fetch_trials_tree({})) == 3
assert len(experiment.fetch_trials({})) == 2
orion.core.cli.main((
"-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_cli").split(" "))
assert len(experiment.fetch_trials_tree({})) == 21
assert len(experiment.fetch_trials({})) == 20
def test_new_algo(init_full_x_new_algo):
"""Test that new algo conflict is automatically resolved"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_new_algo'})
assert len(experiment.fetch_trials(with_evc_tree=True)) == 3
assert len(experiment.fetch_trials()) == 2
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_algo"
).split(" "))
assert len(experiment.fetch_trials(with_evc_tree=True)) == 21
assert len(experiment.fetch_trials()) == 20
def get_experiments(args):
"""Return the different experiments.
Parameters
----------
args: dict
Commandline arguments.
"""
projection = {'name': 1}
query = {'name': args['name']} if args.get('name') else {}
experiments = Database().read("experiments", query, projection)
return [
EVCBuilder().build_view_from({'name': exp['name']})
for exp in experiments
]
def test_run_entire_full_x_full_y(init_entire, create_db_instance):
"""Test if branched experiment can be executed without triggering a branching event again"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
assert len(experiment.fetch_trials_tree({})) == 23
assert len(experiment.fetch_trials({})) == 4
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y "
"./black_box_with_y.py "
"-x~uniform(-10,10) "
"-y~uniform(-10,10,default_value=1)").split(" "))
assert len(experiment.fetch_trials_tree({})) == 39
assert len(experiment.fetch_trials({})) == 20
def get_experiments(args):
"""Return the different experiments.
Parameters
----------
args: dict
Commandline arguments.
"""
projection = {'name': 1, 'version': 1}
query = {'name': args['name']} if args.get('name') else {}
experiments = get_storage().fetch_experiments(query, projection)
return [
EVCBuilder().build_view_from({
'name': exp['name'],
'version': exp.get('version', 1)
}) for exp in experiments
]
def main(args):
"""Build experiment and execute hunt command"""
args['root'] = None
args['leafs'] = []
experiment = EVCBuilder().build_from(args)
workon(experiment)
def main(args):
"""Fetch config and info experiments"""
experiment = EVCBuilder().build_from(args)
print(format_info(experiment))
