def testOptunaReportTooOften(self):
from ray.tune.suggest.optuna import OptunaSearch
from optuna.samplers import RandomSampler
searcher = OptunaSearch(
sampler=RandomSampler(seed=1234),
space=OptunaSearch.convert_search_space(self.config),
metric="metric",
mode="max",
)
searcher.suggest("trial_1")
searcher.on_trial_result("trial_1", {
"training_iteration": 1,
"metric": 1
})
searcher.on_trial_complete("trial_1", {
"training_iteration": 2,
"metric": 1
})
# Report after complete should not fail
searcher.on_trial_result("trial_1", {
"training_iteration": 3,
"metric": 1
})
searcher.on_trial_complete("trial_1", {
"training_iteration": 4,
"metric": 1
})
def testConvertOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch, param
from optuna.samplers import RandomSampler
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = OptunaSearch.convert_search_space(config)
optuna_config = [
param.suggest_categorical("a", [2, 3, 4]),
param.suggest_int("b/x", 0, 5, 2),
param.suggest_loguniform("b/z", 1e-4, 1e-2)
]
sampler1 = RandomSampler(seed=1234)
searcher1 = OptunaSearch(space=converted_config,
sampler=sampler1,
metric="a",
mode="max")
sampler2 = RandomSampler(seed=1234)
searcher2 = OptunaSearch(space=optuna_config,
sampler=sampler2,
metric="a",
mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = OptunaSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix List and Dict
}
searcher = OptunaSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def testOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch
from optuna.trial import TrialState
searcher = OptunaSearch(
space=self.space,
metric="metric",
mode="max",
points_to_evaluate=[{self.param_name: self.valid_value}],
evaluated_rewards=[1.0],
)
get_len = lambda s: len(s._ot_study.trials) # noqa E731
self.assertGreater(get_len(searcher), 0)
searcher = OptunaSearch(
space=self.space,
metric="metric",
mode="max",
)
point = {
self.param_name: self.valid_value,
}
self.assertEqual(get_len(searcher), 0)
searcher.add_evaluated_point(point, 1.0, intermediate_values=[0.8, 0.9])
self.assertEqual(get_len(searcher), 1)
self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.COMPLETE)
searcher.add_evaluated_point(
point, 1.0, intermediate_values=[0.8, 0.9], error=True
)
self.assertEqual(get_len(searcher), 2)
self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.FAIL)
searcher.add_evaluated_point(
point, 1.0, intermediate_values=[0.8, 0.9], pruned=True
)
self.assertEqual(get_len(searcher), 3)
self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.PRUNED)
searcher.suggest("1")
searcher = OptunaSearch(
space=self.space,
metric="metric",
mode="max",
)
self.run_add_evaluated_trials(searcher, get_len, get_len)
def dbr_space(trial):
return {self.param_name: trial.suggest_float(self.param_name, 0.0, 5.0)}
dbr_searcher = OptunaSearch(
space=dbr_space,
metric="metric",
mode="max",
)
with self.assertRaises(TypeError):
dbr_searcher.add_evaluated_point(point, 1.0)
def testConvertOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch, param
import optuna
from optuna.samplers import RandomSampler
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
OptunaSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = OptunaSearch.convert_search_space(config)
optuna_config = {
"a": optuna.distributions.CategoricalDistribution([2, 3, 4]),
"b": {
"x": optuna.distributions.IntUniformDistribution(0, 5, step=2),
"z": optuna.distributions.LogUniformDistribution(1e-4, 1e-2)
}
}
legacy_optuna_config = [
param.suggest_categorical("a", [2, 3, 4]),
param.suggest_int("b/x", 0, 5, 2),
param.suggest_loguniform("b/z", 1e-4, 1e-2)
]
sampler1 = RandomSampler(seed=1234)
searcher1 = OptunaSearch(space=converted_config,
sampler=sampler1,
metric="a",
mode="max")
sampler2 = RandomSampler(seed=1234)
searcher2 = OptunaSearch(space=optuna_config,
sampler=sampler2,
metric="a",
mode="max")
sampler3 = RandomSampler(seed=1234)
searcher3 = OptunaSearch(space=legacy_optuna_config,
sampler=sampler3,
metric="a",
mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
config3 = searcher3.suggest("0")
self.assertEqual(config1, config2)
self.assertEqual(config1, config3)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = OptunaSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix List and Dict
}
searcher = OptunaSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)