def test_experiment_overwritten_evc_disabled(self, parent_version_config, caplog): """Build an existing experiment with different config, overwritting previous config.""" parent_version_config.pop("version") with OrionState(experiments=[parent_version_config]): with caplog.at_level(logging.WARNING): exp = experiment_builder.build(name=parent_version_config["name"]) assert "Running experiment in a different state" not in caplog.text assert exp.version == 1 assert exp.configuration["algorithms"] == {"random": {"seed": None}} with caplog.at_level(logging.WARNING): exp = experiment_builder.build( name=parent_version_config["name"], algorithms="gradient_descent" ) assert "Running experiment in a different state" in caplog.text assert exp.version == 1 assert list(exp.configuration["algorithms"].keys())[0] == "gradient_descent" caplog.clear() with caplog.at_level(logging.WARNING): exp = experiment_builder.load(name=parent_version_config["name"]) assert "Running experiment in a different state" not in caplog.text assert exp.version == 1 assert list(exp.configuration["algorithms"].keys())[0] == "gradient_descent"
def test_hierarchical_space(self, new_config): """Verify space can have hierarchical structure""" space = { "a": { "x": "uniform(0, 10, discrete=True)" }, "b": { "y": "loguniform(1e-08, 1)", "z": "choices(['voici', 'voila', 2])" }, } with OrionState(experiments=[], trials=[]): exp = experiment_builder.build("hierarchy", space=space) exp2 = experiment_builder.build("hierarchy") assert "a.x" in exp.space assert "b.y" in exp.space assert "b.z" in exp.space # Make sure it can be fetched properly from db as well assert "a.x" in exp2.space assert "b.y" in exp2.space assert "b.z" in exp2.space
def test_new_child_with_branch(self): """Check that experiment is not incremented when branching with a new name.""" name = 'parent' space = {'x': 'uniform(0, 10)'} with OrionState(experiments=[], trials=[]): parent = experiment_builder.build(name=name, space=space) assert parent.name == name assert parent.version == 1 child_name = 'child' child = experiment_builder.build( name=name, branching={'branch_to': child_name}) assert child.name == child_name assert child.version == 1 assert child.refers['parent_id'] == parent.id child_name = 'child2' child = experiment_builder.build(name=child_name, branching={'branch_from': name}) assert child.name == child_name assert child.version == 1 assert child.refers['parent_id'] == parent.id
def add_experiment(**kwargs): """Adds experiment to the dummy orion instance""" base_experiment.update(copy.deepcopy(kwargs)) experiment_builder.build( branching=dict(branch_from=base_experiment["name"], enable=True), **base_experiment )
def test_hierarchical_space(self, new_config): """Verify space can have hierarchical structure""" space = { 'a': { 'x': 'uniform(0, 10, discrete=True)' }, 'b': { 'y': 'loguniform(1e-08, 1)', 'z': 'choices([\'voici\', \'voila\', 2])' } } with OrionState(experiments=[], trials=[]): exp = experiment_builder.build('hierarchy', space=space) exp2 = experiment_builder.build('hierarchy') assert 'a.x' in exp.space assert 'b.y' in exp.space assert 'b.z' in exp.space # Make sure it can be fetched properly from db as well assert 'a.x' in exp2.space assert 'b.y' in exp2.space assert 'b.z' in exp2.space
def three_experiments_same_name_with_trials(two_experiments_same_name, storage): """Create three experiments with the same name but different versions.""" orion.core.cli.main([ "hunt", "--init-only", "-n", "test_single_exp", "./black_box.py", "--x~uniform(0,1)", "--y~normal(0,1)", "--z~+normal(0,1)", ]) ensure_deterministic_id("test_single_exp", storage, version=3) exp = experiment_builder.build(name="test_single_exp", version=1) exp2 = experiment_builder.build(name="test_single_exp", version=2) exp3 = experiment_builder.build(name="test_single_exp", version=3) x = {"name": "/x", "type": "real"} y = {"name": "/y", "type": "real"} z = {"name": "/z", "type": "real"} x_value = 0 for status in Trial.allowed_stati: x["value"] = x_value y["value"] = x_value * 10 z["value"] = x_value * 100 trial = Trial(experiment=exp.id, params=[x], status=status) trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) trial3 = Trial(experiment=exp3.id, params=[x, y, z], status=status) Database().write("trials", trial.to_dict()) Database().write("trials", trial2.to_dict()) Database().write("trials", trial3.to_dict()) x_value += 1
def test_new_child_with_branch(self): """Check that experiment is not incremented when branching with a new name.""" name = "parent" space = {"x": "uniform(0, 10)"} with OrionState(experiments=[], trials=[]): parent = experiment_builder.build(name=name, space=space) assert parent.name == name assert parent.version == 1 child_name = "child" child = experiment_builder.build( name=name, branching={"branch_to": child_name, "enable": True} ) assert child.name == child_name assert child.version == 1 assert child.refers["parent_id"] == parent.id child_name = "child2" child = experiment_builder.build( name=child_name, branching={"branch_from": name, "enable": True} ) assert child.name == child_name assert child.version == 1 assert child.refers["parent_id"] == parent.id
def test_working_dir_works_when_db_absent(self, database, new_config): """Check if working_dir is correctly when absent from the database.""" with OrionState(experiments=[], trials=[]): exp = experiment_builder.build(**new_config) storage = get_storage() found_config = list( storage.fetch_experiments( {"name": "supernaekei", "metadata.user": "******"} ) ) found_config = found_config[0] exp = experiment_builder.build(**found_config) assert exp.working_dir == ""
def test_working_dir_is_correctly_set(self, new_config): """Check if working_dir is correctly changed.""" with OrionState(): new_config['working_dir'] = './' exp = experiment_builder.build(**new_config) storage = get_storage() found_config = list( storage.fetch_experiments({ 'name': 'supernaekei', 'metadata.user': '******' })) found_config = found_config[0] exp = experiment_builder.build(**found_config) assert exp.working_dir == './'
def test_working_dir_is_correctly_set(self, new_config): """Check if working_dir is correctly changed.""" with OrionState(): new_config["working_dir"] = "./" exp = experiment_builder.build(**new_config) storage = get_storage() found_config = list( storage.fetch_experiments( {"name": "supernaekei", "metadata.user": "******"} ) ) found_config = found_config[0] exp = experiment_builder.build(**found_config) assert exp.working_dir == "./"
def test_load_unavailable_algo(algo_unavailable_config, capsys): with OrionState(experiments=[algo_unavailable_config]): experiment = experiment_builder.load("supernaekei", mode="r") assert experiment.algorithms == algo_unavailable_config["algorithms"] assert (experiment.configuration["algorithms"] == algo_unavailable_config["algorithms"]) experiment = experiment_builder.load("supernaekei", mode="w") assert experiment.algorithms == algo_unavailable_config["algorithms"] assert (experiment.configuration["algorithms"] == algo_unavailable_config["algorithms"]) with pytest.raises(NotImplementedError) as exc: experiment_builder.build("supernaekei") exc.match("Could not find implementation of BaseAlgorithm")
def test_load_unavailable_strategy(strategy_unavailable_config, capsys): with OrionState(experiments=[strategy_unavailable_config]): experiment = experiment_builder.load("supernaekei", mode="r") assert experiment.producer == strategy_unavailable_config["producer"] assert (experiment.configuration["producer"] == strategy_unavailable_config["producer"]) experiment = experiment_builder.load("supernaekei", mode="w") assert experiment.producer == strategy_unavailable_config["producer"] assert (experiment.configuration["producer"] == strategy_unavailable_config["producer"]) with pytest.raises(NotImplementedError) as exc: experiment_builder.build("supernaekei") exc.match("Could not find implementation of BaseParallelStrategy")
def test_code_changed(config, monkeypatch): """Check that trial has its working_dir attribute changed.""" exp = experiment_builder.build(**config) trial = tuple_to_trial((1.0,), exp.space) exp.register_trial(trial, status="reserved") con = Consumer(exp) def code_changed(user_script): return dict( type="git", is_dirty=True, HEAD_sha="changed", active_branch="new_branch", diff_sha="new_diff", ) monkeypatch.setattr(consumer, "infer_versioning_metadata", code_changed) with pytest.raises(BranchingEvent) as exc: con.consume(trial) assert exc.match("Code changed between execution of 2 trials")
def with_experiment_using_python_api(monkeypatch, one_experiment): """Create an experiment without trials.""" experiment = experiment_builder.build( name="from-python-api", space={"x": "uniform(0, 10)"} ) return experiment
def test_new_experiment_wout_version(self, space): """Create a new and never-seen-before experiment without a version.""" with OrionState(): exp = experiment_builder.build(name="exp_wout_version", space=space) assert exp.version == 1
def test_evc_duplicates(monkeypatch, producer): """Verify that producer wont register samples that are available in parent experiment""" experiment = producer.experiment new_experiment = build(experiment.name, algorithms="random", branching={"enable": True}) # Replace parent with hacked exp, otherwise parent ID does not match trials in DB # and fetch_trials() won't return anything. new_experiment._node.parent._item = experiment assert len(new_experiment.fetch_trials(with_evc_tree=True)) == len( experiment.fetch_trials()) trials = experiment.fetch_trials() def suggest(pool_size=None): suggest_trials = [] while trials and len(suggest_trials) < pool_size: suggest_trials.append(trials.pop(0)) return suggest_trials producer.experiment = new_experiment producer.algorithm = new_experiment.algorithms monkeypatch.setattr(new_experiment.algorithms, "suggest", suggest) producer.update() producer.produce(len(trials) + 2) assert len(trials) == 0 assert len(new_experiment.fetch_trials(with_evc_tree=False)) == 0
def create_study_experiments(exp_config, trial_config, algorithms, task_number, max_trial, n_workers=(1, )): gen_exps, gen_trials = generate_benchmark_experiments_trials( algorithms, exp_config, trial_config, task_number * len(n_workers), max_trial) from orion.client.experiment import ExperimentClient from orion.executor.joblib_backend import Joblib workers = [] for _ in range(task_number): for worker in n_workers: for _ in range(len(algorithms)): workers.append(worker) with OrionState(experiments=gen_exps, trials=gen_trials): experiments = [] experiments_info = [] for i in range(task_number * len(n_workers) * len(algorithms)): experiment = experiment_builder.build( "experiment-name-{}".format(i)) executor = Joblib(n_workers=workers[i], backend="threading") client = ExperimentClient(experiment, executor=executor) experiments.append(client) for index, exp in enumerate(experiments): experiments_info.append((int(index / task_number), exp)) yield experiments_info
def test_build_no_hit(config_file, random_dt, script_path): """Try building experiment from config when not in db""" name = "supernaekei" space = {"x": "uniform(0, 10)"} max_trials = 100 max_broken = 5 with OrionState(experiments=[], trials=[]): with pytest.raises(NoConfigurationError) as exc_info: experiment_builder.load(name) assert "No experiment with given name 'supernaekei' and version '*'" in str( exc_info.value ) exp = experiment_builder.build( name, space=space, max_trials=max_trials, max_broken=max_broken ) assert exp.name == name assert exp.configuration["refers"] == { "adapter": [], "parent_id": None, "root_id": exp._id, } assert exp.metadata == { "datetime": random_dt, "user": "******", "orion_version": "XYZ", } assert exp.configuration["space"] == space assert exp.max_trials == max_trials assert exp.max_broken == max_broken assert not exp.is_done assert exp.algorithms.configuration == {"random": {"seed": None}}
def test_build_no_hit(config_file, random_dt, script_path): """Try building experiment from config when not in db""" name = 'supernaekei' space = {'x': 'uniform(0, 10)'} max_trials = 100 with OrionState(experiments=[], trials=[]): with pytest.raises(ValueError) as exc_info: experiment_builder.build_view(name) assert "No experiment with given name 'supernaekei' and version '*'" in str( exc_info.value) exp = experiment_builder.build(name, space=space, max_trials=max_trials) assert exp.name == name assert exp.configuration['refers'] == { 'adapter': [], 'parent_id': None, 'root_id': exp._id } assert exp.metadata == { 'datetime': random_dt, 'user': '******', 'orion_version': 'XYZ' } assert exp.configuration['space'] == space assert exp.max_trials == max_trials assert not exp.is_done assert exp.algorithms.configuration == {'random': {'seed': None}}
def test_good_set_before_init_no_hit(self, random_dt, new_config): """Trying to set, overwrite everything from input.""" with OrionState(experiments=[], trials=[]): exp = experiment_builder.build(**new_config) found_config = list( get_storage().fetch_experiments( {"name": "supernaekei", "metadata.user": "******"} ) ) new_config["metadata"]["datetime"] = exp.metadata["datetime"] assert len(found_config) == 1 _id = found_config[0].pop("_id") assert _id != "fasdfasfa" assert exp._id == _id new_config["refers"] = {} new_config.pop("_id") new_config.pop("something_to_be_ignored") new_config["algorithms"]["dumbalgo"]["done"] = False new_config["algorithms"]["dumbalgo"]["judgement"] = None new_config["algorithms"]["dumbalgo"]["scoring"] = 0 new_config["algorithms"]["dumbalgo"]["suspend"] = False new_config["algorithms"]["dumbalgo"]["value"] = 5 new_config["algorithms"]["dumbalgo"]["seed"] = None new_config["refers"] = {"adapter": [], "parent_id": None, "root_id": _id} assert found_config[0] == new_config assert exp.name == new_config["name"] assert exp.configuration["refers"] == new_config["refers"] assert exp.metadata == new_config["metadata"] assert exp.max_trials == new_config["max_trials"] assert exp.max_broken == new_config["max_broken"] assert exp.working_dir == new_config["working_dir"] assert exp.version == new_config["version"] assert exp.algorithms.configuration == new_config["algorithms"]
def hacked_exp(with_user_dendi, random_dt, clean_db, create_db_instance): """Return an `Experiment` instance with hacked _id to find trials in fake database. """ exp = experiment_builder.build(name="supernaedo2-dendi") exp._id = "supernaedo2-dendi" # white box hack return exp
def test_evc_duplicates(monkeypatch, producer): """Verify that producer wont register samples that are available in parent experiment""" experiment = producer.experiment new_experiment = build(experiment.name, algorithms="random") # Replace parent with hacked exp, otherwise parent ID does not match trials in DB # and fetch_trials() won't return anything. new_experiment._node.parent._item = experiment assert len(new_experiment.fetch_trials(with_evc_tree=True)) == len( experiment.fetch_trials()) def suggest(pool_size): return [ trial_to_tuple(experiment.fetch_trials()[-1], experiment.space) ] producer.experiment = new_experiment producer.algorithm = new_experiment.algorithms producer.max_idle_time = 1 monkeypatch.setattr(new_experiment.algorithms, "suggest", suggest) producer.update() with pytest.raises(SampleTimeout): producer.produce() assert len(new_experiment.fetch_trials(with_evc_tree=False)) == 0
def test_instantiation_after_init(self, new_config): """Verify that algo, space and refers was instanciated properly""" with OrionState(experiments=[new_config], trials=[]): exp = experiment_builder.build(**new_config) assert isinstance(exp.algorithms, SpaceTransformAlgoWrapper) assert isinstance(exp.space, Space) assert isinstance(exp.refers["adapter"], BaseAdapter)
def test_backward_compatibility_no_version(self, parent_version_config): """Branch from parent that has no version field.""" parent_version_config.pop("version") with OrionState(experiments=[parent_version_config]): exp = experiment_builder.build(name=parent_version_config["name"], space={"y": "uniform(0, 10)"}) assert exp.version == 2
def test_algo_case_insensitive(self, new_config): """Verify that algo with uppercase or lowercase leads to same experiment""" with OrionState(experiments=[new_config], trials=[]): new_config["algorithms"]["DUMBALGO"] = new_config[ "algorithms"].pop("dumbalgo") exp = experiment_builder.build(**new_config) assert exp.version == 1
def test_old_experiment_2_wout_version( self, parent_version_config, child_version_config ): """Create an already existing experiment without a version and getting last one.""" with OrionState(experiments=[parent_version_config, child_version_config]): exp = experiment_builder.build(name=parent_version_config["name"]) assert exp.version == 2
def family_with_trials(two_experiments): """Create two related experiments with all types of trials.""" exp = experiment_builder.build(name='test_double_exp') exp2 = experiment_builder.build(name='test_double_exp_child') x = {'name': '/x', 'type': 'real'} y = {'name': '/y', 'type': 'real'} x_value = 0 for status in Trial.allowed_stati: x['value'] = x_value y['value'] = x_value trial = Trial(experiment=exp.id, params=[x], status=status) x['value'] = x_value trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) x_value += 1 Database().write('trials', trial.to_dict()) Database().write('trials', trial2.to_dict())
def family_with_trials(two_experiments): """Create two related experiments with all types of trials.""" exp = experiment_builder.build(name="test_double_exp") exp2 = experiment_builder.build(name="test_double_exp_child") x = {"name": "/x", "type": "real"} y = {"name": "/y", "type": "real"} x_value = 0 for status in Trial.allowed_stati: x["value"] = x_value y["value"] = x_value trial = Trial(experiment=exp.id, params=[x], status=status) x["value"] = x_value trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) x_value += 1 Database().write("trials", trial.to_dict()) Database().write("trials", trial2.to_dict())
def single_with_trials(single_without_success): """Create an experiment with all types of trials.""" exp = experiment_builder.build(name="test_single_exp") x = {"name": "/x", "type": "real", "value": 100} results = {"name": "obj", "type": "objective", "value": 0} trial = Trial(experiment=exp.id, params=[x], status="completed", results=[results]) database_factory.create().write("trials", trial.to_dict()) return exp.configuration
def with_experiment_missing_conf_file(monkeypatch, one_experiment): """Create an experiment without trials.""" exp = experiment_builder.build(name="test_single_exp", version=1) conf_file = "idontexist.yaml" exp.metadata["user_config"] = conf_file exp.metadata["user_args"] += ["--config", conf_file] Database().write("experiments", exp.configuration, query={"_id": exp.id}) return exp