def test_optimize_branin(self): """Test that algorithm optimizes a simple task comparably to random search.""" max_trials = type(self).branin_task_max_trials task = Branin() space = self.create_space(task.get_search_space()) algo = self.create_algo(space=space) algo.algorithm.max_trials = max_trials all_suggested_trials: list[Trial] = [] all_objectives: list[float] = [] # NOTE: Some algos work more effectively if they are asked to produce a batch of trials, # rather than a single trial at a time. max_batch_size = 5 while len(all_suggested_trials) < max_trials and not algo.is_done: trials = algo.suggest(max_batch_size) all_suggested_trials.extend(trials) results = [task(**trial.params) for trial in trials] # NOTE: This is true for the branin task. If we ever test other tasks, this could vary. assert all(len(result) == 1 for result in results) new_objectives = [result[0]["value"] for result in results] all_objectives.extend(new_objectives) # NOTE: Not ideal that we have to unpack and repack the results of the task. results_for_backward_observe = [{ "objective": objective } for objective in new_objectives] backward.algo_observe(algo=algo, trials=trials, results=results_for_backward_observe) assert algo.is_done assert min(all_objectives) <= 10
def test_is_done_cardinality(self): """Test that algorithm will stop when cardinality is reached""" space = self.update_space( { "x": "uniform(0, 4, discrete=True)", "y": "choices(['a', 'b', 'c'])", "z": "loguniform(1, 6, discrete=True)", } ) space = self.create_space(space) assert space.cardinality == 5 * 3 * 6 algo = self.create_algo(space=space) for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))): assert not algo.is_done n = algo.n_suggested backward.algo_observe( algo, [format_trials.tuple_to_trial([x, y, z], space)], [dict(objective=i)], ) assert algo.n_suggested == n + 1 assert i + 1 == space.cardinality assert algo.is_done
def test_is_done_cardinality(self): """Test that algorithm will stop when cardinality is reached""" space = SpaceBuilder().build({ "x": "uniform(0, 4, discrete=True)", "y": "choices(['a', 'b', 'c'])", "z": "loguniform(1, 6, discrete=True)", }) assert space.cardinality == 5 * 3 * 6 algo = self.create_algo(space=space) # Prevent the algo from exiting early because of a max_trials limit. algo.algorithm.max_trials = None i = 0 for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))): assert not algo.is_done n = algo.n_suggested backward.algo_observe( algo, [format_trials.tuple_to_trial([x, y, z], space)], [dict(objective=i)], ) assert algo.n_suggested == n + 1 assert i + 1 == space.cardinality assert algo.is_done
def test_optimize_branin(self): """Test that algorithm optimizes somehow (this is on-par with random search)""" MAX_TRIALS = 20 task = Branin() space = self.create_space(task.get_search_space()) algo = self.create_algo(config={}, space=space) algo.algorithm.max_trials = MAX_TRIALS safe_guard = 0 trials = [] objectives = [] while trials or not algo.is_done: if safe_guard >= MAX_TRIALS: break if not trials: trials = algo.suggest(MAX_TRIALS - len(objectives)) trial = trials.pop(0) results = task(trial.params["x"]) objectives.append(results[0]["value"]) backward.algo_observe(algo, [trial], [dict(objective=objectives[-1])]) safe_guard += 1 assert algo.is_done assert min(objectives) <= 10
def test_is_done_cardinality(monkeypatch, dumbalgo): """Check whether algorithm will stop with base algorithm cardinality check""" monkeypatch.delattr(dumbalgo, "is_done") space = Space() space.register(Integer("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest(6) for i in range(1, 6): backward.algo_observe(algo, [format_trials.tuple_to_trial( (i, ), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 5 assert algo.is_done space = Space() space.register(Real("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest(6) for i in range(1, 6): backward.algo_observe(algo, [format_trials.tuple_to_trial( (i, ), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 5 assert not algo.is_done
def test_is_done_cardinality(self): # TODO: Support correctly loguniform(discrete=True) # See https://github.com/Epistimio/orion/issues/566 space = self.update_space( { "x": "uniform(0, 4, discrete=True)", "y": "choices(['a', 'b', 'c'])", "z": "uniform(1, 6, discrete=True)", } ) space = self.create_space(space) assert space.cardinality == 5 * 3 * 6 algo = self.create_algo(space=space) for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))): assert not algo.is_done n = algo.n_suggested backward.algo_observe( algo, [format_trials.tuple_to_trial([x, y, z], space)], [dict(objective=i)], ) assert algo.n_suggested == n + 1 assert i + 1 == space.cardinality assert algo.is_done
def test_observe(self): """Verify that algorithm observes trial without any issues""" algo = self.create_algo() a = algo.space.sample()[0] backward.algo_observe(algo, [a], [dict(objective=1)]) b = algo.suggest(1)[0] backward.algo_observe(algo, [b], [dict(objective=2)])
def test_has_observed(self): """Verify that algorithm detects correctly if a trial was observed""" algo = self.create_algo() a = algo.suggest(1)[0] assert not algo.has_observed(a) backward.algo_observe(algo, [a], [dict(objective=1)]) assert algo.has_observed(a) b = algo.suggest(1)[0] assert not algo.has_observed(b) backward.algo_observe(algo, [b], [dict(objective=2)]) assert algo.has_observed(b)
def observe_trials(self, trials, algo, objective=0): """Make the algorithm observe trials Parameters ---------- trials: list of ``orion.core.worker.trial.Trial`` Trials formatted as tuples of values algo: ``orion.algo.base.BaseAlgorithm`` The algorithm used to observe trials. objective: int, optional The base objective for the trials. All objectives will have value ``objective + i``. Defaults to 0. """ backward.algo_observe( algo, trials, [dict(objective=objective + i) for i in range(len(trials))] )
def test_optimize_multi_objectives(self): """Test that algorithm optimizes somehow (this is on-par with random search)""" _max_trials = 20 task = BraninCurrinTask() space = self.create_space(task.get_search_space()) algo = self.create_algo(config={ **self.config, "extra_objectives": ["statistic"] }, space=space) algo.algorithm.max_trials = _max_trials safe_guard = 0 trials = [] objectives = [] while trials or not algo.is_done: if safe_guard >= _max_trials: break if not trials: trials = algo.suggest(_max_trials - len(objectives)) trial = trials.pop(0) results = task(trial.params["x"]) objectives.append((results[0]["value"], results[1]["value"])) backward.algo_observe( algo, [trial], [ dict(objective=objectives[-1][0], statistic=objectives[-1][1]) ], ) safe_guard += 1 rand_objectives = [] for trial in algo.space.sample(len(objectives)): results = task(trial.params["x"]) rand_objectives.append((results[0]["value"], results[1]["value"])) objectives_branin, objectives_currin = list(zip(*objectives)) _, rand_objectives_currin = list(zip(*rand_objectives)) assert algo.is_done # branin assert min(objectives_branin) <= 10 # currin assert min(objectives_currin) <= min(rand_objectives_currin)
def test_objectives_constraints(self): """Test that algorithm optimizes somehow (this is on-par with random search)""" _max_trials = 20 task = BraninCurrinTask() space = self.create_space(task.get_search_space()) algo = self.create_algo( config={ **self.config, "constraints": ["constraint >= 3", "constraint <= 10"], }, space=space, ) algo.algorithm.max_trials = _max_trials safe_guard = 0 trials = [] objectives = [] while trials or not algo.is_done: if safe_guard >= _max_trials: break if not trials: trials = algo.suggest(_max_trials - len(objectives)) trial = trials.pop(0) results = task(trial.params["x"]) objectives.append((results[0]["value"], results[1]["value"])) backward.algo_observe( algo, [trial], [ dict(objective=objectives[-1][0], constraint=objectives[-1][1]) ], ) safe_guard += 1 objectives_branin, objectives_currin = list(zip(*objectives)) assert algo.is_done # branin assert (min(objectives_branin) <= stats.mean(objectives_branin) - stats.stdev(objectives_branin) * 0.7) # currin assert 3 <= stats.mean(objectives_currin[-5:]) <= 10
def test_state_dict(dumbalgo): """Check whether trials_info is in the state dict""" space = Space() dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) dim = Real("yolo3", "alpha", 0.9) space.register(dim) nested_algo = {"DumbAlgo": dict(value=6, scoring=5)} algo = dumbalgo(space, value=(1, 1)) algo.suggest(1) assert not algo.state_dict["registry"]["_trials"] backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 1 backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 1
def test_is_done_max_trials(monkeypatch, dumbalgo): """Check whether algorithm will stop with base algorithm max_trials check""" monkeypatch.delattr(dumbalgo, "is_done") space = Space() space.register(Real("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest(5) for i in range(1, 5): backward.algo_observe(algo, [format_trials.tuple_to_trial( (i, ), space)], [dict(objective=3)]) assert len(algo.state_dict["_trials_info"]) == 4 assert not algo.is_done dumbalgo.max_trials = 4 assert algo.is_done
def test_has_observed_statedict(self): """Verify that algorithm detects correctly if a trial was observed even when state was restored.""" algo = self.create_algo() a = algo.suggest(1)[0] backward.algo_observe(algo, [a], [dict(objective=1)]) state = algo.state_dict algo = self.create_algo() assert not algo.has_observed(a) algo.set_state(state) assert algo.has_observed(a) b = algo.suggest(1)[0] backward.algo_observe(algo, [b], [dict(objective=2)]) state = algo.state_dict algo = self.create_algo() assert not algo.has_observed(b) algo.set_state(state) assert algo.has_observed(b)
def test_observe(self, palgo, fixed_suggestion): """Observe wraps observations.""" backward.algo_observe(palgo, [fixed_suggestion], [dict(objective=5)]) palgo.observe([fixed_suggestion]) assert palgo.algorithm._trials[0].params == fixed_suggestion.params