def _get_optuna_params(self): from ray.tune.suggest.optuna import param config_space = [] for param_name, space in self.param_distributions.items(): prior = "uniform" param_name = str(param_name) if isinstance(space, tuple) and len(space) >= 2 and len(space) <= 3: try: low = float(space[0]) high = float(space[1]) except Exception: raise ValueError( "low and high need to be of type float, " f"are of type {type(low)} and {type(high)}") from None if len(space) == 3: prior = space[2] if prior not in ["uniform", "log-uniform"]: raise ValueError( "prior needs to be either " f"'uniform' or 'log-uniform', was {prior}") if prior == "log-uniform": config_space.append( param.suggest_loguniform(param_name, low, high)) else: config_space.append( param.suggest_uniform(param_name, low, high)) elif isinstance(space, list): config_space.append( param.suggest_categorical(param_name, space)) else: config_space.append(space) return config_space
def testConvertOptuna(self): from ray.tune.suggest.optuna import OptunaSearch, param from optuna.samplers import RandomSampler config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = OptunaSearch.convert_search_space(config) optuna_config = [ param.suggest_categorical("a", [2, 3, 4]), param.suggest_int("b/x", 0, 5, 2), param.suggest_loguniform("b/z", 1e-4, 1e-2) ] sampler1 = RandomSampler(seed=1234) searcher1 = OptunaSearch(space=converted_config, sampler=sampler1, metric="a", mode="max") sampler2 = RandomSampler(seed=1234) searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = OptunaSearch(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4] mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix List and Dict } searcher = OptunaSearch(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)
# Feed the score back back to Tune. tune.report(iterations=step, mean_loss=intermediate_score) time.sleep(0.1) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument( "--smoke-test", action="store_true", help="Finish quickly for testing") args, _ = parser.parse_known_args() ray.init(configure_logging=False) space = [ param.suggest_uniform("width", 0, 20), param.suggest_uniform("height", -100, 100), # This is an ignored parameter. param.suggest_categorical("activation", ["relu", "tanh"]) ] config = { "num_samples": 10 if args.smoke_test else 100, "config": { "steps": 100, } } algo = OptunaSearch(space, metric="mean_loss", mode="min") scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min") tune.run(easy_objective, search_alg=algo, scheduler=scheduler, **config)
def testConvertOptuna(self): from ray.tune.suggest.optuna import OptunaSearch, param import optuna from optuna.samplers import RandomSampler # Grid search not supported, should raise ValueError with self.assertRaises(ValueError): OptunaSearch.convert_search_space( {"grid": tune.grid_search([0, 1])}) config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = OptunaSearch.convert_search_space(config) optuna_config = { "a": optuna.distributions.CategoricalDistribution([2, 3, 4]), "b": { "x": optuna.distributions.IntUniformDistribution(0, 5, step=2), "z": optuna.distributions.LogUniformDistribution(1e-4, 1e-2) } } legacy_optuna_config = [ param.suggest_categorical("a", [2, 3, 4]), param.suggest_int("b/x", 0, 5, 2), param.suggest_loguniform("b/z", 1e-4, 1e-2) ] sampler1 = RandomSampler(seed=1234) searcher1 = OptunaSearch(space=converted_config, sampler=sampler1, metric="a", mode="max") sampler2 = RandomSampler(seed=1234) searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2, metric="a", mode="max") sampler3 = RandomSampler(seed=1234) searcher3 = OptunaSearch(space=legacy_optuna_config, sampler=sampler3, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") config3 = searcher3.suggest("0") self.assertEqual(config1, config2) self.assertEqual(config1, config3) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = OptunaSearch(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4] mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix List and Dict } searcher = OptunaSearch(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)