def test_fetch_config_from_db_hit(new_config):
"""Verify db config when experiment is in db"""
with OrionState(experiments=[new_config], trials=[]):
db_config = experiment_builder.fetch_config_from_db(name="supernaekei")
assert db_config["name"] == new_config["name"]
assert db_config["refers"] == new_config["refers"]
assert db_config["metadata"] == new_config["metadata"]
assert db_config["pool_size"] == new_config["pool_size"]
assert db_config["max_trials"] == new_config["max_trials"]
assert db_config["max_broken"] == new_config["max_broken"]
assert db_config["algorithms"] == new_config["algorithms"]
assert db_config["metadata"] == new_config["metadata"]
def test_reserve_trial_fail(self, storage):
"""Test reserve trial"""
with OrionState(
experiments=[base_experiment],
trials=generate_trials(status=["completed", "reserved"]),
storage=storage,
) as cfg:
storage = cfg.storage()
experiment = cfg.get_experiment("default_name", version=None)
trial = storage.reserve_trial(experiment)
assert trial is None
def test_fix_lost_trials(self):
"""Test that a running trial with an old heartbeat is set to interrupted."""
trial = copy.deepcopy(base_trial)
trial["status"] = "reserved"
trial["heartbeat"] = datetime.datetime.utcnow() - datetime.timedelta(
seconds=60 * 10)
with OrionState(trials=[trial]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
assert len(exp.fetch_trials_by_status("reserved")) == 1
exp.fix_lost_trials()
assert len(exp.fetch_trials_by_status("reserved")) == 0
def check_status_change(new_status):
with OrionState(
experiments=[base_experiment], trials=generate_trials(), storage=storage
) as cfg:
trial = get_storage().get_trial(cfg.get_trial(0))
assert trial is not None, "Was not able to retrieve trial for test"
assert trial.status != new_status
if trial.status == new_status:
return
with pytest.raises(FailedUpdate):
trial.status = new_status
get_storage().set_trial_status(trial, status=new_status)
def test_create_experiment_hit_no_config(self):
"""Test creating an existing experiment by specifying the name only."""
with OrionState(experiments=[config]):
experiment = create_experiment(config["name"])
assert experiment.name == config["name"]
assert experiment.version == 1
assert experiment.space.configuration == config["space"]
assert experiment.algorithms.configuration == config["algorithms"]
assert experiment.max_trials == config["max_trials"]
assert experiment.max_broken == config["max_broken"]
assert experiment.working_dir == config["working_dir"]
assert (experiment.producer["strategy"].configuration ==
config["producer"]["strategy"])
def test_fetch_non_completed_trials():
"""Fetch a list of the trials that are not completed
trials.status in ['new', 'interrupted', 'suspended', 'broken']
"""
non_completed_stati = ["new", "interrupted", "suspended", "reserved"]
stati = non_completed_stati + ["completed"]
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
trials = exp.fetch_noncompleted_trials()
assert len(trials) == 4
assert set(trial.status for trial in trials) == set(non_completed_stati)
def test_create_with_not_exist_targets_parameters(self, benchmark_config):
"""Test creation with not existing assessment parameters"""
benchmark_config["targets"][0]["assess"]["AverageResult"] = {
"task_num": 2,
"idontexist": 100,
}
with OrionState(benchmarks=benchmark_config):
with pytest.raises(TypeError) as exc:
get_or_create_benchmark(benchmark_config["name"])
assert "__init__() got an unexpected keyword argument 'idontexist'" in str(
exc.value
)
def test_create_experiment_hit_no_branch(self, user_config):
"""Test creating an existing experiment by specifying all identical attributes."""
with OrionState(experiments=[config]):
experiment = create_experiment(**user_config)
exp_config = experiment.configuration
assert experiment.name == config["name"]
assert experiment.version == 1
assert exp_config["space"] == config["space"]
assert exp_config["max_trials"] == config["max_trials"]
assert exp_config["max_broken"] == config["max_broken"]
assert exp_config["working_dir"] == config["working_dir"]
assert exp_config["algorithms"] == config["algorithms"]
def test_setup_studies(self, benchmark):
"""Test to setup studies for benchmark"""
with OrionState():
benchmark.setup_studies()
assert (
str(benchmark.studies)
== "[Study(assessment=AverageResult, task=RosenBrock, algorithms=[random,tpe]), "
"Study(assessment=AverageResult, task=CarromTable, algorithms=[random,tpe]), "
"Study(assessment=AverageRank, task=RosenBrock, algorithms=[random,tpe]), "
"Study(assessment=AverageRank, task=CarromTable, algorithms=[random,tpe])]"
)
assert len(benchmark.studies) == 4
def test_get_from_args_hit(config_file, random_dt, new_config):
"""Try building experiment view when in db"""
cmdargs = {"name": "supernaekei", "config": config_file}
with OrionState(experiments=[new_config], trials=[]):
exp_view = experiment_builder.get_from_args(cmdargs)
assert exp_view._id == new_config["_id"]
assert exp_view.name == new_config["name"]
assert exp_view.configuration["refers"] == new_config["refers"]
assert exp_view.metadata == new_config["metadata"]
assert exp_view.max_trials == new_config["max_trials"]
assert exp_view.max_broken == new_config["max_broken"]
assert exp_view.algorithms.configuration == new_config["algorithms"]
def test_delete_algorithm_lock(self, storage):
if storage and storage["type"] == "track":
pytest.xfail("Track does not support algorithm lock yet.")
with OrionState(experiments=generate_experiments(),
storage=storage) as cfg:
storage = cfg.storage()
experiments = storage.fetch_experiments({})
assert storage.delete_algorithm_lock(
uid=experiments[0]["_id"]) == 1
assert storage.get_algorithm_lock_info(
uid=experiments[0]["_id"]) is None
def test_working_dir_works_when_db_absent(self, database, new_config):
"""Check if working_dir is correctly when absent from the database."""
with OrionState(experiments=[], trials=[]):
exp = experiment_builder.build(**new_config)
storage = get_storage()
found_config = list(
storage.fetch_experiments(
{"name": "supernaekei", "metadata.user": "******"}
)
)
found_config = found_config[0]
exp = experiment_builder.build(**found_config)
assert exp.working_dir == ""
def test_count_completed_trials(self, storage):
"""Test count completed trials"""
with OrionState(
experiments=[base_experiment], trials=generate_trials(), storage=storage
) as cfg:
count = 0
for trial in cfg.trials:
if trial["status"] == "completed":
count += 1
storage = cfg.storage()
experiment = cfg.get_experiment("default_name", version=None)
trials = storage.count_completed_trials(experiment)
assert trials == count
def test_experiment_stats():
"""Check that property stats is returning a proper summary of experiment's results."""
NUM_COMPLETED = 3
stati = (["completed"] * NUM_COMPLETED) + (["reserved"] * 2)
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp.metadata = {"datetime": datetime.datetime.utcnow()}
stats = exp.stats
assert stats.trials_completed == NUM_COMPLETED
assert stats.best_trials_id == cfg.trials[3]["_id"]
assert stats.best_evaluation == 0
assert stats.start_time == exp.metadata["datetime"]
assert stats.finish_time == cfg.trials[0]["end_time"]
assert stats.duration == stats.finish_time - stats.start_time
def check_status_change(new_status):
with OrionState(experiments=[base_experiment],
trials=generate_trials(),
storage=storage) as cfg:
trial = get_storage().get_trial(cfg.get_trial(0))
assert trial is not None, "was not able to retrieve trial for test"
get_storage().set_trial_status(trial, status=new_status)
assert (trial.status == new_status
), "Trial status should have been updated locally"
trial = get_storage().get_trial(trial)
assert (
trial.status == new_status
), "Trial status should have been updated in the storage"
def test_strategy_defined_in_global_config(self, caplog, space,
monkeypatch):
"""Verify there is a warning"""
with monkeypatch.context() as m:
m.setattr(
orion.core.config.experiment,
"strategy",
{"this is deprecated": "and should be ignored"},
)
with OrionState():
with caplog.at_level(logging.WARNING):
exp = experiment_builder.build(name="whatever",
space=space)
assert "`strategy` option is not supported anymore." in caplog.text
def test_load_unavailable_algo(algo_unavailable_config, capsys):
with OrionState(experiments=[algo_unavailable_config]):
experiment = experiment_builder.load("supernaekei", mode="r")
assert experiment.algorithms == algo_unavailable_config["algorithms"]
assert (experiment.configuration["algorithms"] ==
algo_unavailable_config["algorithms"])
experiment = experiment_builder.load("supernaekei", mode="w")
assert experiment.algorithms == algo_unavailable_config["algorithms"]
assert (experiment.configuration["algorithms"] ==
algo_unavailable_config["algorithms"])
with pytest.raises(NotImplementedError) as exc:
experiment_builder.build("supernaekei")
exc.match("Could not find implementation of BaseAlgorithm")
def test_working_dir_is_correctly_set(self, new_config):
"""Check if working_dir is correctly changed."""
with OrionState():
new_config["working_dir"] = "./"
exp = experiment_builder.build(**new_config)
storage = get_storage()
found_config = list(
storage.fetch_experiments(
{"name": "supernaekei", "metadata.user": "******"}
)
)
found_config = found_config[0]
exp = experiment_builder.build(**found_config)
assert exp.working_dir == "./"
def test_load_unavailable_strategy(strategy_unavailable_config, capsys):
with OrionState(experiments=[strategy_unavailable_config]):
experiment = experiment_builder.load("supernaekei", mode="r")
assert experiment.producer == strategy_unavailable_config["producer"]
assert (experiment.configuration["producer"] ==
strategy_unavailable_config["producer"])
experiment = experiment_builder.load("supernaekei", mode="w")
assert experiment.producer == strategy_unavailable_config["producer"]
assert (experiment.configuration["producer"] ==
strategy_unavailable_config["producer"])
with pytest.raises(NotImplementedError) as exc:
experiment_builder.build("supernaekei")
exc.match("Could not find implementation of BaseParallelStrategy")
def test_create_with_invalid_algorithms(self, benchmark_config_py):
"""Test creation with a not existed algorithm"""
with OrionState():
with pytest.raises(NotImplementedError) as exc:
benchmark_config_py["algorithms"] = [{
"algorithm": {
"fake_algorithm": {
"seed": 1
}
}
}]
get_or_create_benchmark(**benchmark_config_py)
assert "Could not find implementation of BaseAlgorithm" in str(
exc.value)
def test_create_with_not_loaded_targets(self, benchmark_config):
"""Test creation with assessment or task does not exist or not loaded"""
cfg_invalid_assess = copy.deepcopy(benchmark_config)
cfg_invalid_assess["targets"][0]["assess"]["idontexist"] = {
"task_num": 2
}
with OrionState(benchmarks=cfg_invalid_assess):
with pytest.raises(NotImplementedError) as exc:
get_or_create_benchmark(benchmark_config["name"])
assert "Could not find implementation of BaseAssess" in str(
exc.value)
cfg_invalid_task = copy.deepcopy(benchmark_config)
cfg_invalid_task["targets"][0]["task"]["idontexist"] = {
"max_trials": 2
}
with OrionState(benchmarks=cfg_invalid_task):
with pytest.raises(NotImplementedError) as exc:
get_or_create_benchmark(benchmark_config["name"])
assert "Could not find implementation of BaseTask" in str(
exc.value)
def test_create_experiment_hit_branch(self):
"""Test creating a differing experiment that cause branching."""
with OrionState(experiments=[config]):
experiment = create_experiment(config["name"],
space={"y": "uniform(0, 10)"})
assert experiment.name == config["name"]
assert experiment.version == 2
assert experiment.algorithms.configuration == config["algorithms"]
assert experiment.max_trials == config["max_trials"]
assert experiment.max_broken == config["max_broken"]
assert experiment.working_dir == config["working_dir"]
assert (experiment.producer["strategy"].configuration ==
config["producer"]["strategy"])
def test_acquire_algorithm_lock_successful(self, storage):
if storage and storage["type"] == "track":
pytest.xfail("Track does not support algorithm lock yet.")
with OrionState(experiments=[base_experiment], storage=storage) as cfg:
storage = cfg.storage()
experiment = cfg.get_experiment("default_name", version=None)
with storage.acquire_algorithm_lock(
experiment, timeout=0.1) as locked_algo_state:
assert locked_algo_state.state is None
locked_algo_state.set_state("my new state")
with storage.acquire_algorithm_lock(
experiment) as locked_algo_state:
assert locked_algo_state.state == "my new state"
def test_acquire_algorithm_lock_timeout(self, storage, mocker):
with OrionState(experiments=[base_experiment], storage=storage) as cfg:
storage = cfg.storage()
experiment = cfg.get_experiment("default_name", version=None)
with storage.acquire_algorithm_lock(
experiment) as locked_algo_state:
retry_interval = 0.2
sleep_mock = mocker.spy(time, "sleep")
with pytest.raises(LockAcquisitionTimeout):
with storage.acquire_algorithm_lock(
experiment,
timeout=0.05,
retry_interval=retry_interval):
pass
def test_push_trial_results(self, storage=None):
"""Successfully push a completed trial into database."""
reserved_trial = copy.deepcopy(base_trial)
reserved_trial["status"] = "reserved"
with OrionState(experiments=[],
trials=[reserved_trial],
storage=storage) as cfg:
storage = cfg.storage()
trial = storage.get_trial(Trial(**reserved_trial))
results = [Trial.Result(name="loss", type="objective", value=2)]
trial.results = results
assert storage.push_trial_results(
trial), "should update successfully"
trial2 = storage.get_trial(trial)
assert trial2.results == results
def test_acquire_algorithm_lock_with_different_config(self, new_config, algorithm):
with OrionState(experiments=[new_config]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = 0
algorithm_original_config = algorithm.configuration
exp.algorithms = algorithm
# Setting attribute to algorithm inside the wrapper
algorithm.algorithm.seed = 10
assert algorithm.configuration != algorithm_original_config
with pytest.raises(
RuntimeError, match="Algorithm configuration changed since"
):
with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1):
pass
def test_acquire_algorithm_lock_timeout(self, new_config, algorithm, mocker):
with OrionState(experiments=[new_config]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = 0
exp.algorithms = algorithm
storage_acquisition_mock = mocker.spy(
cfg.storage(), "acquire_algorithm_lock"
)
with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1):
pass
storage_acquisition_mock.assert_called_with(
experiment=exp, timeout=0.2, retry_interval=0.1
)
def test_create_experiment(self, storage):
"""Test create experiment"""
with OrionState(experiments=[], storage=storage) as cfg:
storage = cfg.storage()
storage.create_experiment(base_experiment)
experiments = storage.fetch_experiments({})
assert len(experiments) == 1, "Only one experiment in the database"
experiment = experiments[0]
assert base_experiment == experiment, "Local experiment and DB should match"
# Insert it again
with pytest.raises(DuplicateKeyError):
storage.create_experiment(base_experiment)
def test_configurable_broken_property():
"""Check if max_broken changes after configuration."""
MAX_BROKEN = 5
stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN))
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp.max_broken = MAX_BROKEN
assert exp.is_broken
exp.max_broken += 1
assert not exp.is_broken
def test_empty(self, space):
"""Test panda frame creation when there is no trials"""
with OrionState():
exp = Experiment("supernaekei", mode="x")
exp.space = space
assert exp.to_pandas().shape == (0, 8)
assert list(exp.to_pandas().columns) == [
"id",
"experiment_id",
"status",
"suggested",
"reserved",
"completed",
"objective",
"/index",
]
