def testOptunaReportTooOften(self): from ray.tune.suggest.optuna import OptunaSearch from optuna.samplers import RandomSampler searcher = OptunaSearch( sampler=RandomSampler(seed=1234), space=OptunaSearch.convert_search_space(self.config), metric="metric", mode="max", ) searcher.suggest("trial_1") searcher.on_trial_result("trial_1", { "training_iteration": 1, "metric": 1 }) searcher.on_trial_complete("trial_1", { "training_iteration": 2, "metric": 1 }) # Report after complete should not fail searcher.on_trial_result("trial_1", { "training_iteration": 3, "metric": 1 }) searcher.on_trial_complete("trial_1", { "training_iteration": 4, "metric": 1 })
def testConvertOptuna(self): from ray.tune.suggest.optuna import OptunaSearch, param from optuna.samplers import RandomSampler config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = OptunaSearch.convert_search_space(config) optuna_config = [ param.suggest_categorical("a", [2, 3, 4]), param.suggest_int("b/x", 0, 5, 2), param.suggest_loguniform("b/z", 1e-4, 1e-2) ] sampler1 = RandomSampler(seed=1234) searcher1 = OptunaSearch(space=converted_config, sampler=sampler1, metric="a", mode="max") sampler2 = RandomSampler(seed=1234) searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = OptunaSearch(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4] mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix List and Dict } searcher = OptunaSearch(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)
def testOptuna(self): from ray.tune.suggest.optuna import OptunaSearch from optuna.trial import TrialState searcher = OptunaSearch( space=self.space, metric="metric", mode="max", points_to_evaluate=[{self.param_name: self.valid_value}], evaluated_rewards=[1.0], ) get_len = lambda s: len(s._ot_study.trials) # noqa E731 self.assertGreater(get_len(searcher), 0) searcher = OptunaSearch( space=self.space, metric="metric", mode="max", ) point = { self.param_name: self.valid_value, } self.assertEqual(get_len(searcher), 0) searcher.add_evaluated_point(point, 1.0, intermediate_values=[0.8, 0.9]) self.assertEqual(get_len(searcher), 1) self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.COMPLETE) searcher.add_evaluated_point( point, 1.0, intermediate_values=[0.8, 0.9], error=True ) self.assertEqual(get_len(searcher), 2) self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.FAIL) searcher.add_evaluated_point( point, 1.0, intermediate_values=[0.8, 0.9], pruned=True ) self.assertEqual(get_len(searcher), 3) self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.PRUNED) searcher.suggest("1") searcher = OptunaSearch( space=self.space, metric="metric", mode="max", ) self.run_add_evaluated_trials(searcher, get_len, get_len) def dbr_space(trial): return {self.param_name: trial.suggest_float(self.param_name, 0.0, 5.0)} dbr_searcher = OptunaSearch( space=dbr_space, metric="metric", mode="max", ) with self.assertRaises(TypeError): dbr_searcher.add_evaluated_point(point, 1.0)
def testConvertOptuna(self): from ray.tune.suggest.optuna import OptunaSearch, param import optuna from optuna.samplers import RandomSampler # Grid search not supported, should raise ValueError with self.assertRaises(ValueError): OptunaSearch.convert_search_space( {"grid": tune.grid_search([0, 1])}) config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = OptunaSearch.convert_search_space(config) optuna_config = { "a": optuna.distributions.CategoricalDistribution([2, 3, 4]), "b": { "x": optuna.distributions.IntUniformDistribution(0, 5, step=2), "z": optuna.distributions.LogUniformDistribution(1e-4, 1e-2) } } legacy_optuna_config = [ param.suggest_categorical("a", [2, 3, 4]), param.suggest_int("b/x", 0, 5, 2), param.suggest_loguniform("b/z", 1e-4, 1e-2) ] sampler1 = RandomSampler(seed=1234) searcher1 = OptunaSearch(space=converted_config, sampler=sampler1, metric="a", mode="max") sampler2 = RandomSampler(seed=1234) searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2, metric="a", mode="max") sampler3 = RandomSampler(seed=1234) searcher3 = OptunaSearch(space=legacy_optuna_config, sampler=sampler3, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") config3 = searcher3.suggest("0") self.assertEqual(config1, config2) self.assertEqual(config1, config3) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = OptunaSearch(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4] mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix List and Dict } searcher = OptunaSearch(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)