def test_build_no_hit(config_file, random_dt, script_path): """Try building experiment from config when not in db""" name = 'supernaekei' space = {'x': 'uniform(0, 10)'} max_trials = 100 with OrionState(experiments=[], trials=[]): with pytest.raises(ValueError) as exc_info: experiment_builder.build_view(name) assert "No experiment with given name 'supernaekei' and version '*'" in str( exc_info.value) exp = experiment_builder.build(name, space=space, max_trials=max_trials) assert exp.name == name assert exp.configuration['refers'] == { 'adapter': [], 'parent_id': None, 'root_id': exp._id } assert exp.metadata == { 'datetime': random_dt, 'user': '******', 'orion_version': 'XYZ' } assert exp.configuration['space'] == space assert exp.max_trials == max_trials assert not exp.is_done assert exp.algorithms.configuration == {'random': {'seed': None}}
def test_empty_experiment_view(self): """Hit user name, but exp_name does not hit the db.""" with OrionState(experiments=[], trials=[]): with pytest.raises(ValueError) as exc_info: experiment_builder.build_view('supernaekei') assert ( "No experiment with given name 'supernaekei' and version '*'" in str(exc_info.value))
def get_experiments(args): """Return the different experiments. Parameters ---------- args: dict Commandline arguments. """ projection = {'name': 1, 'version': 1, 'refers': 1} query = {'name': args['name']} if args.get('name') else {} experiments = get_storage().fetch_experiments(query, projection) if args['name']: root_experiments = experiments else: root_experiments = [ exp for exp in experiments if exp['refers'].get('root_id', exp['_id']) == exp['_id'] ] return [ experiment_builder.build_view(name=exp['name'], version=exp.get('version', 1)) for exp in root_experiments ]
def print_evc(experiments, version=None, all_trials=False, collapse=False, expand_versions=False, **kwargs): """Print each EVC tree Parameters ---------- args: dict Commandline arguments. """ for exp in experiments: experiment = experiment_builder.build_view(exp.name, version) if version is None: expand_experiment = exp else: expand_experiment = experiment expand = expand_versions or _has_named_children(expand_experiment) if expand and not collapse: print_status_recursively(expand_experiment, all_trials=all_trials) else: print_status(experiment, all_trials=all_trials, collapse=True)
def main(args): """List all experiments inside database.""" config = experiment_builder.get_cmd_config(args) experiment_builder.setup_storage(config.get('storage')) query = {} if args['name']: query['name'] = args['name'] experiments = get_storage().fetch_experiments(query) if args['name']: root_experiments = experiments else: root_experiments = [ exp for exp in experiments if exp['refers'].get('root_id', exp['_id']) == exp['_id'] ] if not root_experiments: print("No experiment found") return for root_experiment in root_experiments: root = experiment_builder.build_view( name=root_experiment['name'], version=root_experiment.get('version')).node print_tree(root, nameattr='tree_name')
def test_existing_experiment_view(self, new_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" with OrionState(experiments=[new_config], trials=[]): exp = experiment_builder.build_view(name='supernaekei') assert exp._id == new_config['_id'] assert exp.name == new_config['name'] assert exp.configuration['refers'] == new_config['refers'] assert exp.metadata == new_config['metadata'] assert exp.pool_size == new_config['pool_size'] assert exp.max_trials == new_config['max_trials'] assert exp.version == new_config['version'] assert isinstance(exp.refers['adapter'], BaseAdapter) assert exp.algorithms.configuration == new_config['algorithms'] with pytest.raises(AttributeError): exp.this_is_not_in_config = 5 # Test that experiment.update_completed_trial indeed exists exp._experiment.update_completed_trial with pytest.raises(AttributeError): exp.update_completed_trial with pytest.raises(AttributeError): exp.register_trial with pytest.raises(AttributeError): exp.reserve_trial
def test_build_without_config_hit(python_api_config): """Try building experiment without commandline config when in db (no branch)""" name = 'supernaekei' with OrionState(experiments=[python_api_config], trials=[]): # Test that experiment already exists (this should fail otherwise) experiment_builder.build_view(name=name) exp = experiment_builder.build(name=name) assert exp._id == python_api_config['_id'] assert exp.name == python_api_config['name'] assert exp.configuration['refers'] == python_api_config['refers'] assert exp.metadata == python_api_config['metadata'] assert exp.max_trials == python_api_config['max_trials'] assert exp.algorithms.configuration == python_api_config['algorithms']
def test_full_x_remove_y(init_full_x_remove_y, create_db_instance): """Test if full x removed y is properly initialized and can fetch from its 2 parents""" experiment = experiment_builder.build_view(name='full_x_remove_y') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 7), ), (('/x', -7), )) pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True)) assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ), (('/x', 7), ), (('/x', -7), ))
def test_full_x_remove_z(init_full_x_remove_z, create_db_instance): """Test if full x removed z is properly initialized and can fetch from 2 of its 3 parents""" experiment = experiment_builder.build_view(name='full_x_remove_z') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 8), ), (('/x', -8), )) pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True)) # Note that full_x_rename_y_z are filtered out because default_value=1 assert pairs == ((('/x', 0), ), (('/x', 1), ), (('/x', -1), ), (('/x', 8), ), (('/x', -8), ))
def test_half_x_full_y(init_half_x_full_y, create_db_instance): """Test if half x full y is properly initialized and can fetch from its 2 parents""" experiment = experiment_builder.build_view(name='half_x_full_y') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 2), ('/y', 2)), (('/x', 2), ('/y', -2))) pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True)) assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)), (('/x', 1), ('/y', -1)), (('/x', 2), ('/y', 2)), (('/x', 2), ('/y', -2)))
def test_full_x_full_y(init_full_x_full_y, create_db_instance): """Test if full x full y is properly initialized and can fetch original trial""" experiment = experiment_builder.build_view(name='full_x_full_y') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)), (('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1))) pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True)) assert pairs == ((('/x', 0), ('/y', 1)), (('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)), (('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)))
def test_new_cli(init_full_x_new_cli): """Test that new cli conflict is automatically resolved""" experiment = experiment_builder.build_view(name='full_x_new_cli') assert len(experiment.fetch_trials(with_evc_tree=True)) == 3 assert len(experiment.fetch_trials()) == 2 orion.core.cli.main(( "-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_cli").split(" ")) assert len(experiment.fetch_trials(with_evc_tree=True)) == 21 assert len(experiment.fetch_trials()) == 20
def test_run_entire_full_x_full_y_no_args(init_entire, create_db_instance): """Test if branched experiment can be executed without script arguments""" experiment = experiment_builder.build_view(name='full_x_full_y') assert len(experiment.fetch_trials(with_evc_tree=True)) == 23 assert len(experiment.fetch_trials()) == 4 orion.core.cli.main( ("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y").split(" ")) assert len(experiment.fetch_trials(with_evc_tree=True)) == 39 assert len(experiment.fetch_trials()) == 20
def item(self): """Get the experiment associated to the node Note that accessing `item` may trigger the lazy initialization of the experiment if it was not done already. """ if self._item is None: # TODO: Find another way around the circular import import orion.core.io.experiment_builder as experiment_builder self._item = experiment_builder.build_view(name=self.name, version=self.version) self._item._experiment._node = self return self._item
def test_full_x_rename_half_y_half_z(init_full_x_rename_half_y_half_z, create_db_instance): """Test if full x half z is properly initialized and can fetch from its 3 parents""" experiment = experiment_builder.build_view( name='full_x_rename_half_y_half_z') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 5), ('/z', 5)), (('/x', -5), ('/z', 5))) pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True)) assert pairs == ((('/x', 0), ('/z', 1)), (('/x', 1), ('/z', 1)), (('/x', -1), ('/z', 1)), (('/x', 3), ('/z', 3)), (('/x', -3), ('/z', 3)), (('/x', 5), ('/z', 5)), (('/x', -5), ('/z', 5)))
def test_run_entire_full_x_full_y(init_entire, create_db_instance): """Test if branched experiment can be executed without triggering a branching event again""" experiment = experiment_builder.build_view(name='full_x_full_y') assert len(experiment.fetch_trials(with_evc_tree=True)) == 23 assert len(experiment.fetch_trials()) == 4 orion.core.cli.main( ("-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y " "./black_box_with_y.py " "-x~uniform(-10,10) " "-y~uniform(-10,10,default_value=1)").split(" ")) assert len(experiment.fetch_trials(with_evc_tree=True)) == 39 assert len(experiment.fetch_trials()) == 20
def test_full_x_remove_z_default_4(init_full_x_remove_z_default_4, create_db_instance): """Test if full x removed z (default 4) is properly initialized and can fetch from 1 of its 3 parents """ experiment = experiment_builder.build_view( name='full_x_remove_z_default_4') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 9), ), (('/x', -9), )) pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True)) # Note that full_x and full_x_full_y are filtered out because default_value=4 assert pairs == ((('/x', 4), ), (('/x', -4), ), (('/x', 9), ), (('/x', -9), ))
def test_entire_full_x_full_y(init_entire, create_db_instance): """Test if full x full y can fetch from its parent and all children""" experiment = experiment_builder.build_view(name='full_x_full_y') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)), (('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1))) pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True)) assert set(pairs) == set(( (('/x', 0), ('/y', 1)), # full_x_full_y (('/x', 1), ('/y', 1)), (('/x', -1), ('/y', 1)), (('/x', 1), ('/y', -1)), (('/x', -1), ('/y', -1)), # half_x_full_y (('/x', 2), ('/y', 2)), (('/x', 2), ('/y', -2)), # full_x_half_y (('/x', 3), ('/y', 3)), (('/x', -3), ('/y', 3)), # full_x_rename_y_z (('/x', 4), ('/y', 4)), (('/x', -4), ('/y', 4)), (('/x', 4), ('/y', -4)), (('/x', -4), ('/y', -4)), # full_x_rename_half_y_half_z (('/x', 5), ('/y', 5)), (('/x', -5), ('/y', 5)), # full_x_rename_half_y_full_z (('/x', 6), ('/y', 6)), (('/x', -6), ('/y', 6)), # full_x_remove_y (('/x', 7), ('/y', 1)), (('/x', -7), ('/y', 1)), # full_x_remove_z (('/x', 8), ('/y', 1)), (('/x', -8), ('/y', 1)), # full_x_remove_z_default_4 (('/x', 9), ('/y', 4)), (('/x', -9), ('/y', 4))))
def test_init(init_full_x, create_db_instance): """Test if original experiment contains trial 0""" experiment = experiment_builder.build_view(name='full_x') pairs = get_name_value_pairs(experiment.fetch_trials()) assert pairs == ((('/x', 0), ), )