def test_build_view_from_args_debug_mode(script_path):
"""Try building experiment view in debug mode"""
update_singletons()
# Can't build view if none exist. It's fine we only want to test the storage creation.
with pytest.raises(ValueError):
experiment_builder.build_view_from_args({'name': 'whatever'})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
# Can't build view if none exist. It's fine we only want to test the storage creation.
with pytest.raises(ValueError):
experiment_builder.build_view_from_args({
'name': 'whatever',
'debug': True
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_build_from_args_debug_mode(script_path):
"""Try building experiment in debug mode"""
update_singletons()
experiment_builder.build_from_args({
'name':
'whatever',
'user_args': [script_path, '--mini-batch~uniform(32, 256)']
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
experiment_builder.build_from_args({
'name':
'whatever',
'user_args': [script_path, '--mini-batch~uniform(32, 256)'],
'debug':
True
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_get_storage_uninitiated():
"""Test that get storage fails if no storage singleton exist"""
update_singletons()
with pytest.raises(SingletonNotInstantiatedError) as exc:
get_storage()
assert exc.match("No singleton instance of \(type: Storage\) was created")
def test_release_version(three_experiments_same_name_with_trials):
"""Test releasing a specific experiment version"""
experiments = get_storage().fetch_experiments({})
storage = get_storage()
assert len(experiments) == 3
assert len(storage._fetch_trials({})) > 0
uid = None
for experiment in experiments:
if experiment["version"] == 2:
uid = experiment["_id"]
assert storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 0
assert uid is not None
with storage.acquire_algorithm_lock(uid=uid):
assert storage.get_algorithm_lock_info(uid=uid).locked == 1
for experiment in experiments:
if experiment["version"] == 2:
assert (
storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 1
)
else:
assert (
storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 0
)
execute("db release -f test_single_exp --version 2")
for experiment in experiments:
assert storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 0
def test_create_experiment_debug_mode(self, tmp_path, benchmark_config_py):
"""Test that EphemeralDB is used in debug mode whatever the storage config given"""
update_singletons()
conf_file = str(tmp_path / "db.pkl")
config = copy.deepcopy(benchmark_config_py)
config["storage"] = {
"type": "legacy",
"database": {"type": "pickleddb", "host": conf_file},
}
get_or_create_benchmark(**config)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
config["storage"] = {"type": "legacy", "database": {"type": "pickleddb"}}
config["debug"] = True
get_or_create_benchmark(**config)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_release_name(three_family_branch_with_trials):
"""Test that deleting an experiment removes all children"""
experiments = get_storage().fetch_experiments({})
storage = get_storage()
assert len(experiments) == 3
assert len(storage._fetch_trials({})) > 0
uid = None
for experiment in experiments:
if experiment["name"] == "test_double_exp_child":
uid = experiment["_id"]
assert storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 0
assert uid is not None
with storage.acquire_algorithm_lock(uid=uid):
assert storage.get_algorithm_lock_info(uid=uid).locked == 1
for experiment in experiments:
if experiment["name"] == "test_double_exp_child":
assert (
storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 1
)
else:
assert (
storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 0
)
execute("db release -f test_double_exp_child")
for experiment in experiments:
assert storage.get_algorithm_lock_info(uid=experiment["_id"]).locked == 0
def test_insert_with_version(create_db_instance, monkeypatch, script_path):
"""Try to insert a single trial inside a specific version"""
monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__)))
orion.core.cli.main([
"init_only", "-n", "experiment", "-c", "./orion_config_random.yaml",
script_path, "-x~normal(0,1)"
])
orion.core.cli.main([
"init_only", "-n", "experiment", "-c", "./orion_config_random.yaml",
script_path, "-x~normal(0,1)", "-y~+normal(0,1)"
])
exp = list(get_storage().fetch_experiments({
"name": "experiment",
"version": 1
}))
assert len(exp) == 1
exp = exp[0]
assert '_id' in exp
trials = list(get_storage().fetch_trials(uid=exp['_id']))
assert len(trials) == 0
orion.core.cli.main([
"insert", "-n", "experiment", "--version", "1", "-c",
"./orion_config_random.yaml", script_path, "-x=1"
])
trials = list(get_storage().fetch_trials(uid=exp['_id']))
assert len(trials) == 1
def test_rm_trials_by_status(single_with_trials):
"""Test that trials can be deleted by status"""
trials = get_storage()._fetch_trials({})
n_broken = sum(trial.status == "broken" for trial in trials)
assert n_broken > 0
execute("db rm -f test_single_exp --status broken")
assert len(get_storage()._fetch_trials({})) == len(trials) - n_broken
def check_env_var_config(self, tmp_path, monkeypatch):
"""Check that env vars overrides global configuration"""
update_singletons()
assert orion.core.config.storage.to_dict() == {
"type": self.env_vars["ORION_STORAGE_TYPE"],
"database": {
"name": self.env_vars["ORION_DB_NAME"],
"type": self.env_vars["ORION_DB_TYPE"],
"host": self.env_vars["ORION_DB_ADDRESS"],
"port": int(self.env_vars["ORION_DB_PORT"]),
},
}
with pytest.raises(SingletonNotInstantiatedError):
get_storage()
command = f"hunt --exp-max-trials 0 -n test python {script} -x~uniform(0,1)"
orion.core.cli.main(command.split(" "))
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == os.path.abspath(
self.env_vars["ORION_DB_ADDRESS"])
def test_missing_config(init_full_x_new_config, monkeypatch):
"""Test that experiment can branch with new config if previous is not present"""
name = "full_x_new_config"
experiment = experiment_builder.load(name=name)
# Mess with DB to change config path
metadata = experiment.metadata
bad_config_file = "ho_ho_idontexist.yaml"
config_file = metadata["parser"]["file_config_path"]
metadata["parser"]["file_config_path"] = bad_config_file
metadata["parser"]["parser"]["arguments"][2][1] = bad_config_file
metadata["user_args"][3] = bad_config_file
get_storage().update_experiment(experiment, metadata=metadata)
orion.core.cli.main((
"hunt --enable-evc --init-only -n {name} "
"--cli-change-type noeffect "
"--config-change-type unsure "
"./black_box_new.py -x~uniform(-10,10) --config {config_file}").format(
name=name, config_file=config_file).split(" "))
new_experiment = experiment_builder.load(name=name)
assert new_experiment.version == experiment.version + 1
assert new_experiment.refers["adapter"].configuration == [{
"change_type":
"noeffect",
"of_type":
"commandlinechange"
}]
def test_rm_under_evc(three_family_branch_with_trials):
"""Test that deleting an experiment removes all children"""
assert len(get_storage().fetch_experiments({})) == 3
assert len(get_storage()._fetch_trials({})) > 0
execute("db rm -f test_double_exp_child --version 1")
assert len(get_storage().fetch_experiments({})) == 1
assert len(get_storage()._fetch_trials({})) > 0
def test_rm_all_evc(three_family_branch_with_trials):
"""Test that deleting root removes all experiments"""
assert len(get_storage().fetch_experiments({})) == 3
assert len(get_storage()._fetch_trials({})) > 0
execute("db rm -f test_double_exp --version 1")
assert len(get_storage().fetch_experiments({})) == 0
assert len(get_storage()._fetch_trials({})) == 0
def test_one_exp(single_with_trials):
"""Test that one exp is deleted properly"""
assert len(get_storage().fetch_experiments({})) == 1
assert len(get_storage()._fetch_trials({})) > 0
execute("db rm -f test_single_exp")
assert len(get_storage().fetch_experiments({})) == 0
assert len(get_storage()._fetch_trials({})) == 0
def test_new_script(init_full_x, monkeypatch):
"""Test that experiment can branch with new script path even if previous is not present"""
name = "full_x"
experiment = experiment_builder.load(name=name)
# Mess with DB to change script path
metadata = experiment.metadata
metadata["user_script"] = "oh_oh_idontexist.py"
metadata["user_args"][0] = "oh_oh_idontexist.py"
metadata["parser"]["parser"]["arguments"][0][1] = "oh_oh_idontexist.py"
get_storage().update_experiment(experiment, metadata=metadata)
orion.core.cli.main((
"hunt --enable-evc --init-only -n {name} --config orion_config.yaml ./black_box.py "
"-x~uniform(-10,10) --some-new args").format(name=name).split(" "))
new_experiment = experiment_builder.load(name=name)
assert new_experiment.version == experiment.version + 1
assert new_experiment.refers["adapter"].configuration == [{
"change_type":
"break",
"of_type":
"commandlinechange"
}]
def test_build_from_args_debug_mode(script_path):
"""Try building experiment in debug mode"""
update_singletons()
experiment_builder.build_from_args(
{
"name": "whatever",
"user_args": [script_path, "--mini-batch~uniform(32, 256)"],
}
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
experiment_builder.build_from_args(
{
"name": "whatever",
"user_args": [script_path, "--mini-batch~uniform(32, 256)"],
"debug": True,
}
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_get_storage():
"""Test that get storage gets the singleton"""
update_singletons()
setup_storage({"database": {"type": "pickleddb", "host": "test.pkl"}})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert get_storage() == storage
def test_get_storage():
"""Test that get storage gets the singleton"""
update_singletons()
setup_storage({'database': {'type': 'pickleddb', 'host': 'test.pkl'}})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert get_storage() == storage
def test_observe(self):
"""Verify that `observe()` will update the storage"""
with create_experiment(config, base_trial) as (cfg, experiment, client):
trial = Trial(**cfg.trials[1])
assert trial.results == []
client.reserve(trial)
assert get_storage().get_trial(trial).objective is None
client.observe(trial, [dict(name="objective", type="objective", value=101)])
assert get_storage().get_trial(trial).objective.value == 101
def test_one_exp(single_with_trials):
"""Test that one exp is deleted properly"""
experiments = get_storage().fetch_experiments({})
uid = experiments[0]["_id"]
assert get_storage().get_algorithm_lock_info(uid=uid).locked == 0
with get_storage().acquire_algorithm_lock(uid=uid):
assert get_storage().get_algorithm_lock_info(uid=uid).locked == 1
execute("db release -f test_single_exp")
assert get_storage().get_algorithm_lock_info(uid=uid).locked == 0
def check_status_change(new_status):
with OrionState(from_yaml=exp_config_file,
database=storage) as cfg:
trial = cfg.get_trial(1)
assert trial is not None, 'Was not able to retrieve trial for test'
assert trial.status != new_status
with pytest.raises(FailedUpdate):
trial.status = new_status
get_storage().set_trial_status(trial, status=new_status)
def _fetch_benchmark(name):
if name:
configs = get_storage().fetch_benchmark({"name": name})
else:
configs = get_storage().fetch_benchmark({})
if not configs:
return {}
return configs[0]
def trial(exp):
"""Return a Trial which is registered in DB."""
trial = tuple_to_trial((1.0, ), exp.space)
heartbeat = datetime.datetime.utcnow()
trial.experiment = exp.id
trial.status = "reserved"
trial.heartbeat = heartbeat
get_storage().register_trial(trial)
return trial
def test_change_status_invalid(self, storage):
"""Attempt to change the status of a Trial with an invalid one"""
with OrionState(
experiments=[base_experiment], trials=generate_trials(), storage=storage
) as cfg:
trial = get_storage().get_trial(cfg.get_trial(0))
assert trial is not None, "Was not able to retrieve trial for test"
with pytest.raises(ValueError) as exc:
get_storage().set_trial_status(trial, status="moo")
assert exc.match("Given status `moo` not one of")
def check_status_change(new_status):
with OrionState(from_yaml=exp_config_file,
database=storage) as cfg:
trial = cfg.get_trial(0)
assert trial is not None, 'was not able to retrieve trial for test'
get_storage().set_trial_status(trial, status=new_status)
assert trial.status == new_status, \
'Trial status should have been updated locally'
trial = get_storage().get_trial(trial)
assert trial.status == new_status, \
'Trial status should have been updated in the storage'
def test_observe_under_with(self):
with create_experiment(config, base_trial) as (cfg, experiment, client):
with client.suggest() as trial:
assert trial.status == "reserved"
assert trial.results == []
assert get_storage().get_trial(trial).objective is None
client.observe(
trial, [dict(name="objective", type="objective", value=101)]
)
assert get_storage().get_trial(trial).objective.value == 101
assert trial.status == "completed"
assert trial.status == "completed" # Still completed after __exit__
def one_experiment_changed_vcs(one_experiment):
"""Create an experiment without trials."""
experiment = experiment_builder.build(name=one_experiment["name"])
experiment.metadata["VCS"] = {
"type": "git",
"is_dirty": False,
"HEAD_sha": "new",
"active_branch": "master",
"diff_sha": None,
}
get_storage().update_experiment(experiment, metadata=experiment.metadata)
def test_update_no_match_query(single_with_trials, capsys):
"""Test that no trials are updated when there is no match"""
trials = get_storage()._fetch_trials({})
trials = dict(zip((trial.id for trial in trials), trials))
execute("db set -f test_single_exp status=invalid status=interrupted")
for trial in get_storage()._fetch_trials({}):
assert (
trials[trial.id].status == trial.status
), "status should not have been changed"
captured = capsys.readouterr()
assert captured.out.endswith("0 trials modified\n")
def one_experiment_changed_vcs(one_experiment):
"""Create an experiment without trials."""
experiment = experiment_builder.build(name=one_experiment['name'])
experiment.metadata['VCS'] = {
'type': 'git',
'is_dirty': False,
'HEAD_sha': 'new',
'active_branch': 'master',
'diff_sha': None
}
get_storage().update_experiment(experiment, metadata=experiment.metadata)
def test_missing_and_new_config(init_full_x_new_config, monkeypatch):
"""Test that experiment can branch with new config if previous is not present, with correct
diff.
"""
name = "full_x_new_config"
experiment = experiment_builder.load(name=name)
# Mess with DB to change config path
metadata = experiment.metadata
bad_config_file = "ho_ho_idontexist.yaml"
config_file = metadata["parser"]["file_config_path"]
metadata["parser"]["file_config_path"] = bad_config_file
metadata["parser"]["parser"]["arguments"][2][1] = bad_config_file
metadata["user_args"][3] = bad_config_file
with open(config_file, "w") as f:
f.write(
yaml.dump({
"new_arg": "some-new-value",
"y": "orion~uniform(-10, 20, default_value=0)",
}))
get_storage().update_experiment(experiment, metadata=metadata)
orion.core.cli.main((
"hunt --enable-evc --init-only -n {name} "
"--cli-change-type noeffect "
"--config-change-type unsure "
"./black_box_new.py -x~uniform(-10,10) --config {config_file}").format(
name=name, config_file=config_file).split(" "))
new_experiment = experiment_builder.load(name=name)
assert new_experiment.version == experiment.version + 1
assert new_experiment.refers["adapter"].configuration == [
{
"name": "/y",
"new_prior": "uniform(-10, 20, default_value=0)",
"of_type": "dimensionpriorchange",
"old_prior": "uniform(-10, 10, default_value=0)",
},
{
"change_type": "noeffect",
"of_type": "commandlinechange"
},
{
"change_type": "unsure",
"of_type": "scriptconfigchange"
},
]
def _update_experiment(experiment):
"""Update experiment configuration in database"""
log.debug('updating experiment (name: %s)', experiment.name)
config = experiment.configuration
# TODO: Remove since this should not occur anymore without metadata.user in the indices?
# Writing the final config to an already existing experiment raises
# a DuplicatKeyError because of the embedding id `metadata.user`.
# To avoid this `final_config["name"]` is popped out before
# `db.write()`, thus seamingly breaking the compound index
# `(name, metadata.user)`
config.pop("name")
get_storage().update_experiment(experiment, **config)
