def test_full_x_remove_y(init_full_x_remove_y, create_db_instance):
"""Test if full x removed y is properly initialized and can fetch from its 2 parents"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_remove_y'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 7), ), (('/x', -7), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ),
(('/x', 7), ), (('/x', -7), ))
def test_full_x_remove_z(init_full_x_remove_z, create_db_instance):
"""Test if full x removed z is properly initialized and can fetch from 2 of its 3 parents"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_remove_z'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 8), ), (('/x', -8), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
# Note that full_x_rename_y_z are filtered out because default_value=1
assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ),
(('/x', 8), ), (('/x', -8), ))
def test_half_x_full_y(init_half_x_full_y, create_db_instance):
"""Test if half x full y is properly initialized and can fetch from its 2 parents"""
experiment = EVCBuilder().build_view_from({'name': 'half_x_full_y'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 2), ('/y', 2)), (('/x', 2), ('/y', -2)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', 2), ('/y', 2)),
(('/x', 2), ('/y', -2)))
def test_new_cli(init_full_x_new_cli):
"""Test that new cli conflict is automatically resolved"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_new_cli'})
assert len(experiment.fetch_trials_tree({})) == 3
assert len(experiment.fetch_trials({})) == 2
orion.core.cli.main((
"-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_cli").split(" "))
assert len(experiment.fetch_trials_tree({})) == 21
assert len(experiment.fetch_trials({})) == 20
def test_run_entire_full_x_full_y_no_args(init_entire, create_db_instance):
"""Test if branched experiment can be executed without script arguments"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
assert len(experiment.fetch_trials_tree({})) == 23
assert len(experiment.fetch_trials({})) == 4
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y").split(" "))
assert len(experiment.fetch_trials_tree({})) == 39
assert len(experiment.fetch_trials({})) == 20
def test_new_algo(init_full_x_new_algo):
"""Test that new algo conflict is automatically resolved"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_new_algo'})
assert len(experiment.fetch_trials(with_evc_tree=True)) == 3
assert len(experiment.fetch_trials()) == 2
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_algo"
).split(" "))
assert len(experiment.fetch_trials(with_evc_tree=True)) == 21
assert len(experiment.fetch_trials()) == 20
def test_full_x_full_y(init_full_x_full_y, create_db_instance):
"""Test if full x full y is properly initialized and can fetch original trial"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)))
# pytest.set_trace()
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)), (('/x', 1), ('/y', -1)),
(('/x', -1), ('/y', -1)))
def test_full_x_rename_half_y_half_z(init_full_x_rename_half_y_half_z,
create_db_instance):
"""Test if full x half z is properly initialized and can fetch from its 3 parents"""
experiment = EVCBuilder().build_view_from(
{'name': 'full_x_rename_half_y_half_z'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 5), ('/z', 5)), (('/x', -5), ('/z', 5)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == ((('/x', 0), ('/z', 1)), (('/x', 1), ('/z', 1)),
(('/x', -1), ('/z', 1)), (('/x', 3), ('/z', 3)),
(('/x', -3), ('/z', 3)), (('/x', 5), ('/z', 5)),
(('/x', -5), ('/z', 5)))
def test_run_entire_full_x_full_y(init_entire, create_db_instance):
"""Test if branched experiment can be executed without triggering a branching event again"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
assert len(experiment.fetch_trials_tree({})) == 23
assert len(experiment.fetch_trials({})) == 4
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y "
"./black_box_with_y.py "
"-x~uniform(-10,10) "
"-y~uniform(-10,10,default_value=1)").split(" "))
assert len(experiment.fetch_trials_tree({})) == 39
assert len(experiment.fetch_trials({})) == 20
def test_full_x_remove_z_default_4(init_full_x_remove_z_default_4,
create_db_instance):
"""Test if full x removed z (default 4) is properly initialized and can fetch
from 1 of its 3 parents
"""
experiment = EVCBuilder().build_view_from(
{'name': 'full_x_remove_z_default_4'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 9), ), (('/x', -9), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
# Note that full_x and full_x_full_y are filtered out because default_value=4
assert pairs == ((('/x', 4), ), (('/x', -4), ), (('/x', 9), ), (('/x',
-9), ))
def test_full_x_half_y(init_full_x_half_y, create_db_instance):
"""Test if full x half y is properly initialized and can fetch from its 2 parents"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_half_y'})
pairs = get_name_value_pairs(experiment.fetch_trials({}))
assert pairs == ((('/x', 3), ('/y', 3)), (('/x', -3), ('/y', 3)))
pairs = get_name_value_pairs(experiment.fetch_trials_tree({}))
assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)), (('/x', 3), ('/y', 3)),
(('/x', -3), ('/y', 3)))
def test_entire_full_x_full_y(init_entire, create_db_instance):
"""Test if full x full y can fetch from its parent and all children"""
experiment = EVCBuilder().build_view_from({'name': 'full_x_full_y'})
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert set(pairs) == set((
(('/x', 0), ('/y', 1)),
# full_x_full_y
(('/x', 1), ('/y', 1)),
(('/x', -1), ('/y', 1)),
(('/x', 1), ('/y', -1)),
(('/x', -1), ('/y', -1)),
# half_x_full_y
(('/x', 2), ('/y', 2)),
(('/x', 2), ('/y', -2)),
# full_x_half_y
(('/x', 3), ('/y', 3)),
(('/x', -3), ('/y', 3)),
# full_x_rename_y_z
(('/x', 4), ('/y', 4)),
(('/x', -4), ('/y', 4)),
(('/x', 4), ('/y', -4)),
(('/x', -4), ('/y', -4)),
# full_x_rename_half_y_half_z
(('/x', 5), ('/y', 5)),
(('/x', -5), ('/y', 5)),
# full_x_rename_half_y_full_z
(('/x', 6), ('/y', 6)),
(('/x', -6), ('/y', 6)),
# full_x_remove_y
(('/x', 7), ('/y', 1)),
(('/x', -7), ('/y', 1)),
# full_x_remove_z
(('/x', 8), ('/y', 1)),
(('/x', -8), ('/y', 1)),
# full_x_remove_z_default_4
(('/x', 9), ('/y', 4)),
(('/x', -9), ('/y', 4))))