def test_missing_fidelity(algorithm):
"""Test a simple usage scenario."""
with pytest.raises(RuntimeError) as exc:
workon(rosenbrock, space, algorithms=algorithm, max_trials=100)
assert "https://orion.readthedocs.io/en/develop/user/algorithms.html" in str(
exc.value)
def test_missing_fidelity(algorithm: dict):
"""Test a simple usage scenario."""
task = CustomRosenbrock(max_trials=30, with_fidelity=False)
with pytest.raises(RuntimeError) as exc:
workon(task,
task.get_search_space(),
algorithms=algorithm,
max_trials=100)
assert "https://orion.readthedocs.io/en/develop/user/algorithms.html" in str(
exc.value)
def test_with_multidim(algorithm):
"""Test a scenario with a dimension shape > 1."""
space = copy.deepcopy(space_with_fidelity)
space["x"] = "uniform(-50, 50, shape=(2, 1))"
MAX_TRIALS = 30
exp = workon(multidim_rosenbrock,
space,
algorithms=algorithm,
max_trials=MAX_TRIALS)
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials()
assert len(trials) >= 25 # Grid-search is a bit smaller
completed_trials = exp.fetch_trials_by_status("completed")
assert len(completed_trials) >= MAX_TRIALS or len(completed_trials) == len(
trials)
trials = completed_trials
results = [trial.objective.value for trial in trials]
assert all(trial.objective is not None for trial in trials)
best_trial = min(trials, key=lambda trial: trial.objective.value)
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 10
assert len(best_trial.params) == 2
fidelity = best_trial._params[0]
assert fidelity.name == "noise"
assert fidelity.type == "fidelity"
assert fidelity.value in [1, 2, 5, 10]
param = best_trial._params[1]
assert param.name == "x"
assert param.type == "real"
def test_with_fidelity(algorithm: dict):
"""Test a scenario with fidelity."""
exp = workon(
rosenbrock_with_fidelity,
space_with_fidelity,
algorithms=algorithm,
max_trials=30,
)
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials()
assert len(trials) >= 30
assert trials[29].status == "completed"
trials = [trial for trial in trials if trial.status == "completed"]
results = [trial.objective.value for trial in trials]
assert all(trial.objective is not None for trial in trials)
best_trial = min(trials, key=lambda trial: trial.objective.value)
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 10
assert len(best_trial.params) == 2
fidelity = best_trial._params[0]
assert fidelity.name == "noise"
assert fidelity.type == "fidelity"
assert fidelity.value in [1, 2, 5, 10]
param = best_trial._params[1]
assert param.name == "x"
assert param.type == "real"
def test_cardinality_stop_loguniform(algorithm: dict):
"""Test when algo needs to stop because all space is explored (loguniform space)."""
discrete_space = SpaceBuilder().build(
{"x": "loguniform(0.1, 1, precision=1)"})
max_trials = 30
exp = workon(rosenbrock,
space=discrete_space,
algorithms=algorithm,
max_trials=max_trials)
algo_wrapper: SpaceTransformAlgoWrapper = exp.algorithms
assert algo_wrapper.space == discrete_space
assert algo_wrapper.algorithm.is_done
assert algo_wrapper.is_done
trials = exp.fetch_trials()
if algo_wrapper.algorithm.space.cardinality == 10:
# BUG: See https://github.com/Epistimio/orion/issues/865
# The algo (e.g. GridSearch) believes it has exhausted the space cardinality and exits early
# but that's incorrect! The transformed space should have a different cardinality than the
# original space.
assert len(trials) <= 10
else:
assert len(trials) == 10
assert trials[-1].status == "completed"
def test_with_multidim(algorithm):
"""Test a scenario with a dimension shape > 1."""
space = copy.deepcopy(space_with_fidelity)
space["x"] = "uniform(-50, 50, shape=(3, 2))"
exp = workon(multidim_rosenbrock,
space,
algorithms=algorithm,
max_trials=100)
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials()
assert len(trials) <= 100
assert trials[-1].status == "completed"
results = [trial.objective.value for trial in trials]
best_trial = next(
iter(sorted(trials, key=lambda trial: trial.objective.value)))
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 1e-5
assert len(best_trial.params) == 2
fidelity = best_trial._params[0]
assert fidelity.name == "noise"
assert fidelity.type == "fidelity"
assert fidelity.value == 10
param = best_trial._params[1]
assert param.name == "x"
assert param.type == "real"
def test_with_fidelity(algorithm):
"""Test a scenario with fidelity."""
exp = workon(rosenbrock,
space_with_fidelity,
algorithms=algorithm,
max_trials=100)
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials()
assert len(trials) <= 100
assert trials[-1].status == "completed"
results = [trial.objective.value for trial in trials]
print(min(results))
print(max(results))
best_trial = next(
iter(sorted(trials, key=lambda trial: trial.objective.value)))
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 1e-5
assert len(best_trial.params) == 2
fidelity = best_trial._params[0]
assert fidelity.name == "noise"
assert fidelity.type == "fidelity"
assert fidelity.value == 10
param = best_trial._params[1]
assert param.name == "x"
assert param.type == "real"
def test_cardinality_stop(algorithm):
"""Test when algo needs to stop because all space is explored (dicrete space)."""
discrete_space = copy.deepcopy(space)
discrete_space["x"] = "uniform(-10, 5, discrete=True)"
exp = workon(rosenbrock, discrete_space, algorithms=algorithm, max_trials=30)
trials = exp.fetch_trials()
assert len(trials) == 16
assert trials[-1].status == "completed"
discrete_space["x"] = "loguniform(0.1, 1, precision=1)"
exp = workon(rosenbrock, discrete_space, algorithms=algorithm, max_trials=30)
print(exp.space.cardinality)
trials = exp.fetch_trials()
assert len(trials) == 10
assert trials[-1].status == "completed"
def test_simple(algorithm):
"""Test a simple usage scenario."""
max_trials = 30
exp = workon(rosenbrock, space, algorithms=algorithm, max_trials=max_trials)
assert exp.max_trials == max_trials
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials()
assert len(trials) == max_trials
assert trials[-1].status == "completed"
best_trial = sorted(trials, key=lambda trial: trial.objective.value)[0]
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 15
assert len(best_trial.params) == 1
param = best_trial._params[0]
assert param.name == "x"
assert param.type == "real"