def test_optimize_branin(self):
"""Test that algorithm optimizes a simple task comparably to random search."""
max_trials = type(self).branin_task_max_trials
task = Branin()
space = self.create_space(task.get_search_space())
algo = self.create_algo(space=space)
algo.algorithm.max_trials = max_trials
all_suggested_trials: list[Trial] = []
all_objectives: list[float] = []
# NOTE: Some algos work more effectively if they are asked to produce a batch of trials,
# rather than a single trial at a time.
max_batch_size = 5
while len(all_suggested_trials) < max_trials and not algo.is_done:
trials = algo.suggest(max_batch_size)
all_suggested_trials.extend(trials)
results = [task(**trial.params) for trial in trials]
# NOTE: This is true for the branin task. If we ever test other tasks, this could vary.
assert all(len(result) == 1 for result in results)
new_objectives = [result[0]["value"] for result in results]
all_objectives.extend(new_objectives)
# NOTE: Not ideal that we have to unpack and repack the results of the task.
results_for_backward_observe = [{
"objective": objective
} for objective in new_objectives]
backward.algo_observe(algo=algo,
trials=trials,
results=results_for_backward_observe)
assert algo.is_done
assert min(all_objectives) <= 10
def test_is_done_cardinality(self):
"""Test that algorithm will stop when cardinality is reached"""
space = self.update_space(
{
"x": "uniform(0, 4, discrete=True)",
"y": "choices(['a', 'b', 'c'])",
"z": "loguniform(1, 6, discrete=True)",
}
)
space = self.create_space(space)
assert space.cardinality == 5 * 3 * 6
algo = self.create_algo(space=space)
for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))):
assert not algo.is_done
n = algo.n_suggested
backward.algo_observe(
algo,
[format_trials.tuple_to_trial([x, y, z], space)],
[dict(objective=i)],
)
assert algo.n_suggested == n + 1
assert i + 1 == space.cardinality
assert algo.is_done
def test_is_done_cardinality(self):
"""Test that algorithm will stop when cardinality is reached"""
space = SpaceBuilder().build({
"x": "uniform(0, 4, discrete=True)",
"y": "choices(['a', 'b', 'c'])",
"z": "loguniform(1, 6, discrete=True)",
})
assert space.cardinality == 5 * 3 * 6
algo = self.create_algo(space=space)
# Prevent the algo from exiting early because of a max_trials limit.
algo.algorithm.max_trials = None
i = 0
for i, (x, y,
z) in enumerate(itertools.product(range(5), "abc", range(1,
7))):
assert not algo.is_done
n = algo.n_suggested
backward.algo_observe(
algo,
[format_trials.tuple_to_trial([x, y, z], space)],
[dict(objective=i)],
)
assert algo.n_suggested == n + 1
assert i + 1 == space.cardinality
assert algo.is_done
def test_optimize_branin(self):
"""Test that algorithm optimizes somehow (this is on-par with random search)"""
MAX_TRIALS = 20
task = Branin()
space = self.create_space(task.get_search_space())
algo = self.create_algo(config={}, space=space)
algo.algorithm.max_trials = MAX_TRIALS
safe_guard = 0
trials = []
objectives = []
while trials or not algo.is_done:
if safe_guard >= MAX_TRIALS:
break
if not trials:
trials = algo.suggest(MAX_TRIALS - len(objectives))
trial = trials.pop(0)
results = task(trial.params["x"])
objectives.append(results[0]["value"])
backward.algo_observe(algo, [trial], [dict(objective=objectives[-1])])
safe_guard += 1
assert algo.is_done
assert min(objectives) <= 10
def test_is_done_cardinality(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm cardinality check"""
monkeypatch.delattr(dumbalgo, "is_done")
space = Space()
space.register(Integer("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert algo.is_done
space = Space()
space.register(Real("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert not algo.is_done
def test_is_done_cardinality(self):
# TODO: Support correctly loguniform(discrete=True)
# See https://github.com/Epistimio/orion/issues/566
space = self.update_space(
{
"x": "uniform(0, 4, discrete=True)",
"y": "choices(['a', 'b', 'c'])",
"z": "uniform(1, 6, discrete=True)",
}
)
space = self.create_space(space)
assert space.cardinality == 5 * 3 * 6
algo = self.create_algo(space=space)
for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))):
assert not algo.is_done
n = algo.n_suggested
backward.algo_observe(
algo,
[format_trials.tuple_to_trial([x, y, z], space)],
[dict(objective=i)],
)
assert algo.n_suggested == n + 1
assert i + 1 == space.cardinality
assert algo.is_done
def test_observe(self):
"""Verify that algorithm observes trial without any issues"""
algo = self.create_algo()
a = algo.space.sample()[0]
backward.algo_observe(algo, [a], [dict(objective=1)])
b = algo.suggest(1)[0]
backward.algo_observe(algo, [b], [dict(objective=2)])
def test_has_observed(self):
"""Verify that algorithm detects correctly if a trial was observed"""
algo = self.create_algo()
a = algo.suggest(1)[0]
assert not algo.has_observed(a)
backward.algo_observe(algo, [a], [dict(objective=1)])
assert algo.has_observed(a)
b = algo.suggest(1)[0]
assert not algo.has_observed(b)
backward.algo_observe(algo, [b], [dict(objective=2)])
assert algo.has_observed(b)
def observe_trials(self, trials, algo, objective=0):
"""Make the algorithm observe trials
Parameters
----------
trials: list of ``orion.core.worker.trial.Trial``
Trials formatted as tuples of values
algo: ``orion.algo.base.BaseAlgorithm``
The algorithm used to observe trials.
objective: int, optional
The base objective for the trials. All objectives
will have value ``objective + i``. Defaults to 0.
"""
backward.algo_observe(
algo, trials, [dict(objective=objective + i) for i in range(len(trials))]
)
def test_optimize_multi_objectives(self):
"""Test that algorithm optimizes somehow (this is on-par with random search)"""
_max_trials = 20
task = BraninCurrinTask()
space = self.create_space(task.get_search_space())
algo = self.create_algo(config={
**self.config, "extra_objectives": ["statistic"]
},
space=space)
algo.algorithm.max_trials = _max_trials
safe_guard = 0
trials = []
objectives = []
while trials or not algo.is_done:
if safe_guard >= _max_trials:
break
if not trials:
trials = algo.suggest(_max_trials - len(objectives))
trial = trials.pop(0)
results = task(trial.params["x"])
objectives.append((results[0]["value"], results[1]["value"]))
backward.algo_observe(
algo,
[trial],
[
dict(objective=objectives[-1][0],
statistic=objectives[-1][1])
],
)
safe_guard += 1
rand_objectives = []
for trial in algo.space.sample(len(objectives)):
results = task(trial.params["x"])
rand_objectives.append((results[0]["value"], results[1]["value"]))
objectives_branin, objectives_currin = list(zip(*objectives))
_, rand_objectives_currin = list(zip(*rand_objectives))
assert algo.is_done
# branin
assert min(objectives_branin) <= 10
# currin
assert min(objectives_currin) <= min(rand_objectives_currin)
def test_objectives_constraints(self):
"""Test that algorithm optimizes somehow (this is on-par with random search)"""
_max_trials = 20
task = BraninCurrinTask()
space = self.create_space(task.get_search_space())
algo = self.create_algo(
config={
**self.config,
"constraints": ["constraint >= 3", "constraint <= 10"],
},
space=space,
)
algo.algorithm.max_trials = _max_trials
safe_guard = 0
trials = []
objectives = []
while trials or not algo.is_done:
if safe_guard >= _max_trials:
break
if not trials:
trials = algo.suggest(_max_trials - len(objectives))
trial = trials.pop(0)
results = task(trial.params["x"])
objectives.append((results[0]["value"], results[1]["value"]))
backward.algo_observe(
algo,
[trial],
[
dict(objective=objectives[-1][0],
constraint=objectives[-1][1])
],
)
safe_guard += 1
objectives_branin, objectives_currin = list(zip(*objectives))
assert algo.is_done
# branin
assert (min(objectives_branin) <= stats.mean(objectives_branin) -
stats.stdev(objectives_branin) * 0.7)
# currin
assert 3 <= stats.mean(objectives_currin[-5:]) <= 10
def test_state_dict(dumbalgo):
"""Check whether trials_info is in the state dict"""
space = Space()
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "alpha", 0.9)
space.register(dim)
nested_algo = {"DumbAlgo": dict(value=6, scoring=5)}
algo = dumbalgo(space, value=(1, 1))
algo.suggest(1)
assert not algo.state_dict["registry"]["_trials"]
backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)],
[dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 1
backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)],
[dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 1
def test_is_done_max_trials(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm max_trials check"""
monkeypatch.delattr(dumbalgo, "is_done")
space = Space()
space.register(Real("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(5)
for i in range(1, 5):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["_trials_info"]) == 4
assert not algo.is_done
dumbalgo.max_trials = 4
assert algo.is_done
def test_has_observed_statedict(self):
"""Verify that algorithm detects correctly if a trial was observed even when state was restored."""
algo = self.create_algo()
a = algo.suggest(1)[0]
backward.algo_observe(algo, [a], [dict(objective=1)])
state = algo.state_dict
algo = self.create_algo()
assert not algo.has_observed(a)
algo.set_state(state)
assert algo.has_observed(a)
b = algo.suggest(1)[0]
backward.algo_observe(algo, [b], [dict(objective=2)])
state = algo.state_dict
algo = self.create_algo()
assert not algo.has_observed(b)
algo.set_state(state)
assert algo.has_observed(b)
def test_observe(self, palgo, fixed_suggestion):
"""Observe wraps observations."""
backward.algo_observe(palgo, [fixed_suggestion], [dict(objective=5)])
palgo.observe([fixed_suggestion])
assert palgo.algorithm._trials[0].params == fixed_suggestion.params
