def test_experiment_overwritten_evc_disabled(self, parent_version_config, caplog):
"""Build an existing experiment with different config, overwritting previous config."""
parent_version_config.pop("version")
with OrionState(experiments=[parent_version_config]):
with caplog.at_level(logging.WARNING):
exp = experiment_builder.build(name=parent_version_config["name"])
assert "Running experiment in a different state" not in caplog.text
assert exp.version == 1
assert exp.configuration["algorithms"] == {"random": {"seed": None}}
with caplog.at_level(logging.WARNING):
exp = experiment_builder.build(
name=parent_version_config["name"], algorithms="gradient_descent"
)
assert "Running experiment in a different state" in caplog.text
assert exp.version == 1
assert list(exp.configuration["algorithms"].keys())[0] == "gradient_descent"
caplog.clear()
with caplog.at_level(logging.WARNING):
exp = experiment_builder.load(name=parent_version_config["name"])
assert "Running experiment in a different state" not in caplog.text
assert exp.version == 1
assert list(exp.configuration["algorithms"].keys())[0] == "gradient_descent"
def test_hierarchical_space(self, new_config):
"""Verify space can have hierarchical structure"""
space = {
"a": {
"x": "uniform(0, 10, discrete=True)"
},
"b": {
"y": "loguniform(1e-08, 1)",
"z": "choices(['voici', 'voila', 2])"
},
}
with OrionState(experiments=[], trials=[]):
exp = experiment_builder.build("hierarchy", space=space)
exp2 = experiment_builder.build("hierarchy")
assert "a.x" in exp.space
assert "b.y" in exp.space
assert "b.z" in exp.space
# Make sure it can be fetched properly from db as well
assert "a.x" in exp2.space
assert "b.y" in exp2.space
assert "b.z" in exp2.space
def test_new_child_with_branch(self):
"""Check that experiment is not incremented when branching with a new name."""
name = 'parent'
space = {'x': 'uniform(0, 10)'}
with OrionState(experiments=[], trials=[]):
parent = experiment_builder.build(name=name, space=space)
assert parent.name == name
assert parent.version == 1
child_name = 'child'
child = experiment_builder.build(
name=name, branching={'branch_to': child_name})
assert child.name == child_name
assert child.version == 1
assert child.refers['parent_id'] == parent.id
child_name = 'child2'
child = experiment_builder.build(name=child_name,
branching={'branch_from': name})
assert child.name == child_name
assert child.version == 1
assert child.refers['parent_id'] == parent.id
def add_experiment(**kwargs):
"""Adds experiment to the dummy orion instance"""
base_experiment.update(copy.deepcopy(kwargs))
experiment_builder.build(
branching=dict(branch_from=base_experiment["name"], enable=True),
**base_experiment
)
def test_hierarchical_space(self, new_config):
"""Verify space can have hierarchical structure"""
space = {
'a': {
'x': 'uniform(0, 10, discrete=True)'
},
'b': {
'y': 'loguniform(1e-08, 1)',
'z': 'choices([\'voici\', \'voila\', 2])'
}
}
with OrionState(experiments=[], trials=[]):
exp = experiment_builder.build('hierarchy', space=space)
exp2 = experiment_builder.build('hierarchy')
assert 'a.x' in exp.space
assert 'b.y' in exp.space
assert 'b.z' in exp.space
# Make sure it can be fetched properly from db as well
assert 'a.x' in exp2.space
assert 'b.y' in exp2.space
assert 'b.z' in exp2.space
def three_experiments_same_name_with_trials(two_experiments_same_name,
storage):
"""Create three experiments with the same name but different versions."""
orion.core.cli.main([
"hunt",
"--init-only",
"-n",
"test_single_exp",
"./black_box.py",
"--x~uniform(0,1)",
"--y~normal(0,1)",
"--z~+normal(0,1)",
])
ensure_deterministic_id("test_single_exp", storage, version=3)
exp = experiment_builder.build(name="test_single_exp", version=1)
exp2 = experiment_builder.build(name="test_single_exp", version=2)
exp3 = experiment_builder.build(name="test_single_exp", version=3)
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x], status=status)
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
trial3 = Trial(experiment=exp3.id, params=[x, y, z], status=status)
Database().write("trials", trial.to_dict())
Database().write("trials", trial2.to_dict())
Database().write("trials", trial3.to_dict())
x_value += 1
def test_new_child_with_branch(self):
"""Check that experiment is not incremented when branching with a new name."""
name = "parent"
space = {"x": "uniform(0, 10)"}
with OrionState(experiments=[], trials=[]):
parent = experiment_builder.build(name=name, space=space)
assert parent.name == name
assert parent.version == 1
child_name = "child"
child = experiment_builder.build(
name=name, branching={"branch_to": child_name, "enable": True}
)
assert child.name == child_name
assert child.version == 1
assert child.refers["parent_id"] == parent.id
child_name = "child2"
child = experiment_builder.build(
name=child_name, branching={"branch_from": name, "enable": True}
)
assert child.name == child_name
assert child.version == 1
assert child.refers["parent_id"] == parent.id
def test_working_dir_works_when_db_absent(self, database, new_config):
"""Check if working_dir is correctly when absent from the database."""
with OrionState(experiments=[], trials=[]):
exp = experiment_builder.build(**new_config)
storage = get_storage()
found_config = list(
storage.fetch_experiments(
{"name": "supernaekei", "metadata.user": "******"}
)
)
found_config = found_config[0]
exp = experiment_builder.build(**found_config)
assert exp.working_dir == ""
def test_working_dir_is_correctly_set(self, new_config):
"""Check if working_dir is correctly changed."""
with OrionState():
new_config['working_dir'] = './'
exp = experiment_builder.build(**new_config)
storage = get_storage()
found_config = list(
storage.fetch_experiments({
'name': 'supernaekei',
'metadata.user': '******'
}))
found_config = found_config[0]
exp = experiment_builder.build(**found_config)
assert exp.working_dir == './'
def test_working_dir_is_correctly_set(self, new_config):
"""Check if working_dir is correctly changed."""
with OrionState():
new_config["working_dir"] = "./"
exp = experiment_builder.build(**new_config)
storage = get_storage()
found_config = list(
storage.fetch_experiments(
{"name": "supernaekei", "metadata.user": "******"}
)
)
found_config = found_config[0]
exp = experiment_builder.build(**found_config)
assert exp.working_dir == "./"
def test_load_unavailable_algo(algo_unavailable_config, capsys):
with OrionState(experiments=[algo_unavailable_config]):
experiment = experiment_builder.load("supernaekei", mode="r")
assert experiment.algorithms == algo_unavailable_config["algorithms"]
assert (experiment.configuration["algorithms"] ==
algo_unavailable_config["algorithms"])
experiment = experiment_builder.load("supernaekei", mode="w")
assert experiment.algorithms == algo_unavailable_config["algorithms"]
assert (experiment.configuration["algorithms"] ==
algo_unavailable_config["algorithms"])
with pytest.raises(NotImplementedError) as exc:
experiment_builder.build("supernaekei")
exc.match("Could not find implementation of BaseAlgorithm")
def test_load_unavailable_strategy(strategy_unavailable_config, capsys):
with OrionState(experiments=[strategy_unavailable_config]):
experiment = experiment_builder.load("supernaekei", mode="r")
assert experiment.producer == strategy_unavailable_config["producer"]
assert (experiment.configuration["producer"] ==
strategy_unavailable_config["producer"])
experiment = experiment_builder.load("supernaekei", mode="w")
assert experiment.producer == strategy_unavailable_config["producer"]
assert (experiment.configuration["producer"] ==
strategy_unavailable_config["producer"])
with pytest.raises(NotImplementedError) as exc:
experiment_builder.build("supernaekei")
exc.match("Could not find implementation of BaseParallelStrategy")
def test_code_changed(config, monkeypatch):
"""Check that trial has its working_dir attribute changed."""
exp = experiment_builder.build(**config)
trial = tuple_to_trial((1.0,), exp.space)
exp.register_trial(trial, status="reserved")
con = Consumer(exp)
def code_changed(user_script):
return dict(
type="git",
is_dirty=True,
HEAD_sha="changed",
active_branch="new_branch",
diff_sha="new_diff",
)
monkeypatch.setattr(consumer, "infer_versioning_metadata", code_changed)
with pytest.raises(BranchingEvent) as exc:
con.consume(trial)
assert exc.match("Code changed between execution of 2 trials")
def with_experiment_using_python_api(monkeypatch, one_experiment):
"""Create an experiment without trials."""
experiment = experiment_builder.build(
name="from-python-api", space={"x": "uniform(0, 10)"}
)
return experiment
def test_new_experiment_wout_version(self, space):
"""Create a new and never-seen-before experiment without a version."""
with OrionState():
exp = experiment_builder.build(name="exp_wout_version",
space=space)
assert exp.version == 1
def test_evc_duplicates(monkeypatch, producer):
"""Verify that producer wont register samples that are available in parent experiment"""
experiment = producer.experiment
new_experiment = build(experiment.name,
algorithms="random",
branching={"enable": True})
# Replace parent with hacked exp, otherwise parent ID does not match trials in DB
# and fetch_trials() won't return anything.
new_experiment._node.parent._item = experiment
assert len(new_experiment.fetch_trials(with_evc_tree=True)) == len(
experiment.fetch_trials())
trials = experiment.fetch_trials()
def suggest(pool_size=None):
suggest_trials = []
while trials and len(suggest_trials) < pool_size:
suggest_trials.append(trials.pop(0))
return suggest_trials
producer.experiment = new_experiment
producer.algorithm = new_experiment.algorithms
monkeypatch.setattr(new_experiment.algorithms, "suggest", suggest)
producer.update()
producer.produce(len(trials) + 2)
assert len(trials) == 0
assert len(new_experiment.fetch_trials(with_evc_tree=False)) == 0
def create_study_experiments(exp_config,
trial_config,
algorithms,
task_number,
max_trial,
n_workers=(1, )):
gen_exps, gen_trials = generate_benchmark_experiments_trials(
algorithms, exp_config, trial_config, task_number * len(n_workers),
max_trial)
from orion.client.experiment import ExperimentClient
from orion.executor.joblib_backend import Joblib
workers = []
for _ in range(task_number):
for worker in n_workers:
for _ in range(len(algorithms)):
workers.append(worker)
with OrionState(experiments=gen_exps, trials=gen_trials):
experiments = []
experiments_info = []
for i in range(task_number * len(n_workers) * len(algorithms)):
experiment = experiment_builder.build(
"experiment-name-{}".format(i))
executor = Joblib(n_workers=workers[i], backend="threading")
client = ExperimentClient(experiment, executor=executor)
experiments.append(client)
for index, exp in enumerate(experiments):
experiments_info.append((int(index / task_number), exp))
yield experiments_info
def test_build_no_hit(config_file, random_dt, script_path):
"""Try building experiment from config when not in db"""
name = "supernaekei"
space = {"x": "uniform(0, 10)"}
max_trials = 100
max_broken = 5
with OrionState(experiments=[], trials=[]):
with pytest.raises(NoConfigurationError) as exc_info:
experiment_builder.load(name)
assert "No experiment with given name 'supernaekei' and version '*'" in str(
exc_info.value
)
exp = experiment_builder.build(
name, space=space, max_trials=max_trials, max_broken=max_broken
)
assert exp.name == name
assert exp.configuration["refers"] == {
"adapter": [],
"parent_id": None,
"root_id": exp._id,
}
assert exp.metadata == {
"datetime": random_dt,
"user": "******",
"orion_version": "XYZ",
}
assert exp.configuration["space"] == space
assert exp.max_trials == max_trials
assert exp.max_broken == max_broken
assert not exp.is_done
assert exp.algorithms.configuration == {"random": {"seed": None}}
def test_build_no_hit(config_file, random_dt, script_path):
"""Try building experiment from config when not in db"""
name = 'supernaekei'
space = {'x': 'uniform(0, 10)'}
max_trials = 100
with OrionState(experiments=[], trials=[]):
with pytest.raises(ValueError) as exc_info:
experiment_builder.build_view(name)
assert "No experiment with given name 'supernaekei' and version '*'" in str(
exc_info.value)
exp = experiment_builder.build(name,
space=space,
max_trials=max_trials)
assert exp.name == name
assert exp.configuration['refers'] == {
'adapter': [],
'parent_id': None,
'root_id': exp._id
}
assert exp.metadata == {
'datetime': random_dt,
'user': '******',
'orion_version': 'XYZ'
}
assert exp.configuration['space'] == space
assert exp.max_trials == max_trials
assert not exp.is_done
assert exp.algorithms.configuration == {'random': {'seed': None}}
def test_good_set_before_init_no_hit(self, random_dt, new_config):
"""Trying to set, overwrite everything from input."""
with OrionState(experiments=[], trials=[]):
exp = experiment_builder.build(**new_config)
found_config = list(
get_storage().fetch_experiments(
{"name": "supernaekei", "metadata.user": "******"}
)
)
new_config["metadata"]["datetime"] = exp.metadata["datetime"]
assert len(found_config) == 1
_id = found_config[0].pop("_id")
assert _id != "fasdfasfa"
assert exp._id == _id
new_config["refers"] = {}
new_config.pop("_id")
new_config.pop("something_to_be_ignored")
new_config["algorithms"]["dumbalgo"]["done"] = False
new_config["algorithms"]["dumbalgo"]["judgement"] = None
new_config["algorithms"]["dumbalgo"]["scoring"] = 0
new_config["algorithms"]["dumbalgo"]["suspend"] = False
new_config["algorithms"]["dumbalgo"]["value"] = 5
new_config["algorithms"]["dumbalgo"]["seed"] = None
new_config["refers"] = {"adapter": [], "parent_id": None, "root_id": _id}
assert found_config[0] == new_config
assert exp.name == new_config["name"]
assert exp.configuration["refers"] == new_config["refers"]
assert exp.metadata == new_config["metadata"]
assert exp.max_trials == new_config["max_trials"]
assert exp.max_broken == new_config["max_broken"]
assert exp.working_dir == new_config["working_dir"]
assert exp.version == new_config["version"]
assert exp.algorithms.configuration == new_config["algorithms"]
def hacked_exp(with_user_dendi, random_dt, clean_db, create_db_instance):
"""Return an `Experiment` instance with hacked _id to find trials in
fake database.
"""
exp = experiment_builder.build(name="supernaedo2-dendi")
exp._id = "supernaedo2-dendi" # white box hack
return exp
def test_evc_duplicates(monkeypatch, producer):
"""Verify that producer wont register samples that are available in parent experiment"""
experiment = producer.experiment
new_experiment = build(experiment.name, algorithms="random")
# Replace parent with hacked exp, otherwise parent ID does not match trials in DB
# and fetch_trials() won't return anything.
new_experiment._node.parent._item = experiment
assert len(new_experiment.fetch_trials(with_evc_tree=True)) == len(
experiment.fetch_trials())
def suggest(pool_size):
return [
trial_to_tuple(experiment.fetch_trials()[-1], experiment.space)
]
producer.experiment = new_experiment
producer.algorithm = new_experiment.algorithms
producer.max_idle_time = 1
monkeypatch.setattr(new_experiment.algorithms, "suggest", suggest)
producer.update()
with pytest.raises(SampleTimeout):
producer.produce()
assert len(new_experiment.fetch_trials(with_evc_tree=False)) == 0
def test_instantiation_after_init(self, new_config):
"""Verify that algo, space and refers was instanciated properly"""
with OrionState(experiments=[new_config], trials=[]):
exp = experiment_builder.build(**new_config)
assert isinstance(exp.algorithms, SpaceTransformAlgoWrapper)
assert isinstance(exp.space, Space)
assert isinstance(exp.refers["adapter"], BaseAdapter)
def test_backward_compatibility_no_version(self, parent_version_config):
"""Branch from parent that has no version field."""
parent_version_config.pop("version")
with OrionState(experiments=[parent_version_config]):
exp = experiment_builder.build(name=parent_version_config["name"],
space={"y": "uniform(0, 10)"})
assert exp.version == 2
def test_algo_case_insensitive(self, new_config):
"""Verify that algo with uppercase or lowercase leads to same experiment"""
with OrionState(experiments=[new_config], trials=[]):
new_config["algorithms"]["DUMBALGO"] = new_config[
"algorithms"].pop("dumbalgo")
exp = experiment_builder.build(**new_config)
assert exp.version == 1
def test_old_experiment_2_wout_version(
self, parent_version_config, child_version_config
):
"""Create an already existing experiment without a version and getting last one."""
with OrionState(experiments=[parent_version_config, child_version_config]):
exp = experiment_builder.build(name=parent_version_config["name"])
assert exp.version == 2
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = experiment_builder.build(name='test_double_exp')
exp2 = experiment_builder.build(name='test_double_exp_child')
x = {'name': '/x', 'type': 'real'}
y = {'name': '/y', 'type': 'real'}
x_value = 0
for status in Trial.allowed_stati:
x['value'] = x_value
y['value'] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x['value'] = x_value
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
Database().write('trials', trial2.to_dict())
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = experiment_builder.build(name="test_double_exp")
exp2 = experiment_builder.build(name="test_double_exp_child")
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x["value"] = x_value
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write("trials", trial.to_dict())
Database().write("trials", trial2.to_dict())
def single_with_trials(single_without_success):
"""Create an experiment with all types of trials."""
exp = experiment_builder.build(name="test_single_exp")
x = {"name": "/x", "type": "real", "value": 100}
results = {"name": "obj", "type": "objective", "value": 0}
trial = Trial(experiment=exp.id, params=[x], status="completed", results=[results])
database_factory.create().write("trials", trial.to_dict())
return exp.configuration
def with_experiment_missing_conf_file(monkeypatch, one_experiment):
"""Create an experiment without trials."""
exp = experiment_builder.build(name="test_single_exp", version=1)
conf_file = "idontexist.yaml"
exp.metadata["user_config"] = conf_file
exp.metadata["user_args"] += ["--config", conf_file]
Database().write("experiments", exp.configuration, query={"_id": exp.id})
return exp
