def testOptuna(self): from ray.tune.suggest.optuna import OptunaSearch searcher = OptunaSearch(space=self.config, metric=self.metric_name, mode="max") self._save(searcher) searcher = OptunaSearch(space=self.config, metric=self.metric_name, mode="max") self._restore(searcher)
def testOptuna(self): from ray.tune.suggest.optuna import OptunaSearch from optuna.trial import TrialState searcher = OptunaSearch( space=self.space, metric="metric", mode="max", points_to_evaluate=[{ self.param_name: self.valid_value }], evaluated_rewards=[1.0]) self.assertGreater(len(searcher._ot_study.trials), 0) searcher = OptunaSearch( space=self.space, metric="metric", mode="max", ) point = { self.param_name: self.valid_value, } self.assertEqual(len(searcher._ot_study.trials), 0) searcher.add_evaluated_point( point, 1.0, intermediate_values=[0.8, 0.9]) self.assertEqual(len(searcher._ot_study.trials), 1) self.assertTrue( searcher._ot_study.trials[-1].state == TrialState.COMPLETE) searcher.add_evaluated_point( point, 1.0, intermediate_values=[0.8, 0.9], error=True) self.assertEqual(len(searcher._ot_study.trials), 2) self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.FAIL) searcher.add_evaluated_point( point, 1.0, intermediate_values=[0.8, 0.9], pruned=True) self.assertEqual(len(searcher._ot_study.trials), 3) self.assertTrue( searcher._ot_study.trials[-1].state == TrialState.PRUNED) def dbr_space(trial): return { self.param_name: trial.suggest_float(self.param_name, 0.0, 5.0) } dbr_searcher = OptunaSearch( space=dbr_space, metric="metric", mode="max", ) with self.assertRaises(TypeError): dbr_searcher.add_evaluated_point(point, 1.0)
def testConvertOptuna(self): from ray.tune.suggest.optuna import OptunaSearch, param from optuna.samplers import RandomSampler config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = OptunaSearch.convert_search_space(config) optuna_config = [ param.suggest_categorical("a", [2, 3, 4]), param.suggest_int("b/x", 0, 5, 2), param.suggest_loguniform("b/z", 1e-4, 1e-2) ] sampler1 = RandomSampler(seed=1234) searcher1 = OptunaSearch(space=converted_config, sampler=sampler1, metric="a", mode="max") sampler2 = RandomSampler(seed=1234) searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = OptunaSearch(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4] mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix List and Dict } searcher = OptunaSearch(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)
def set_algorithm(experiment_name, config): ''' Configure search algorithm. ''' if args.algorithm == 'hyperopt': algorithm = HyperOptSearch(points_to_evaluate=best_params) elif args.algorithm == 'ax': ax_client = AxClient(enforce_sequential_optimization=False) ax_client.create_experiment(name=experiment_name, parameters=config, objective_name="minimum", minimize=True) algorithm = AxSearch(ax_client=ax_client, points_to_evaluate=best_params) elif args.algorithm == 'nevergrad': algorithm = NevergradSearch( points_to_evaluate=best_params, optimizer=ng.optimizers.registry["PortfolioDiscreteOnePlusOne"]) elif args.algorithm == 'optuna': algorithm = OptunaSearch(points_to_evaluate=best_params, seed=args.seed) elif args.algorithm == 'pbt': algorithm = PopulationBasedTraining( time_attr="training_iteration", perturbation_interval=args.perturbation, hyperparam_mutations=config, synch=True) elif args.algorithm == 'random': algorithm = BasicVariantGenerator(max_concurrent=args.jobs) if args.algorithm not in ['random', 'pbt']: algorithm = ConcurrencyLimiter(algorithm, max_concurrent=args.jobs) return algorithm
def tune_fn(): mlflow.set_experiment(experiment_name=experiment_name) optuna_search = OptunaSearch(metric="auroc", mode="max") ax_search = AxSearch(metric="auroc", mode="max") tune.run(objective, name="mlflow_gbdt", num_samples=65, config={ "num_leaves": tune.randint(5, 95), "learning_rate": tune.loguniform(1e-4, 1.0), "n_estimators": tune.randint(100, 100000), "subsample": tune.loguniform(0.01, 1.0), "subsample_freq": tune.randint(1, 5), "objective": "binary", "reg_alpha": tune.loguniform(1e-4, 1.0), "reg_lambda": tune.loguniform(1e-4, 1.0), "tree_learner": "feature", "feature_sel": 0, "mlflow": { "experiment_name": experiment_name, "tracking_uri": mlflow.get_tracking_uri() } }, search_alg=optuna_search)
def testOptunaReportTooOften(self): from ray.tune.suggest.optuna import OptunaSearch from optuna.samplers import RandomSampler searcher = OptunaSearch( sampler=RandomSampler(seed=1234), space=OptunaSearch.convert_search_space(self.config), metric="metric", mode="max", ) searcher.suggest("trial_1") searcher.on_trial_result("trial_1", { "training_iteration": 1, "metric": 1 }) searcher.on_trial_complete("trial_1", { "training_iteration": 2, "metric": 1 }) # Report after complete should not fail searcher.on_trial_result("trial_1", { "training_iteration": 3, "metric": 1 }) searcher.on_trial_complete("trial_1", { "training_iteration": 4, "metric": 1 })
def set_basic_conf(self): from optuna.samplers import TPESampler space = [ ot_param.suggest_uniform("width", 0, 20), ot_param.suggest_uniform("height", -100, 100) ] def cost(space, reporter): reporter(loss=(space["height"] - 14)**2 - abs(space["width"] - 3)) search_alg = OptunaSearch( space, sampler=TPESampler(seed=10), metric="loss", mode="min") return search_alg, cost
def run_tune(method: str, num_samples: int) -> tune.ExperimentAnalysis: optuna_search = OptunaSearch(metric="mean_accuracy", mode="max") return tune.run( trial, config=methods[method], num_samples=num_samples, search_alg=optuna_search, # resources_per_trial={"gpu": 1, "cpu": 16}, # Using the GPU makes the search process a lot slower on my machine verbose=1, metric= "mean_accuracy", # otherwise I cannot get the dataframe in the end for some reason mode="max" # See above )
def set_basic_conf(self): from optuna.samplers import TPESampler space = OptunaSearch.convert_search_space({ "width": tune.uniform(0, 20), "height": tune.uniform(-100, 100) }) def cost(space, reporter): reporter(loss=(space["height"] - 14)**2 - abs(space["width"] - 3)) search_alg = OptunaSearch( space, sampler=TPESampler(seed=10), metric="loss", mode="min") return search_alg, cost
def init_search_algorithm(search_alg, metric=None, mode=None): """Specify a search algorithm and you must pip install it first. See more details here: https://docs.ray.io/en/master/tune/api_docs/suggestion.html """ if search_alg == 'optuna': assert metric and mode, "Metric and mode cannot be None for optuna." from ray.tune.suggest.optuna import OptunaSearch return OptunaSearch(metric=metric, mode=mode) elif search_alg == 'bayesopt': assert metric and mode, "Metric and mode cannot be None for bayesian optimization." from ray.tune.suggest.bayesopt import BayesOptSearch return BayesOptSearch(metric=metric, mode=mode) logging.info(f'{search_alg} search is found, run BasicVariantGenerator().')
def testConvergenceOptuna(self): from ray.tune.suggest.optuna import OptunaSearch np.random.seed(1) searcher = OptunaSearch(seed=1) analysis = self._testConvergence( searcher, top=5, ) # This assertion is much weaker than in the BO case, but TPE # don't converge too close. It is still unlikely to get to this # tolerance with random search (5 * 0.1 = 0.5% chance) assert len(analysis.trials) < 100 assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-1)
def run_optuna_tune(smoke_test=False): algo = OptunaSearch( space=define_by_run_func, metric="mean_loss", mode="min") algo = ConcurrencyLimiter(algo, max_concurrent=4) scheduler = AsyncHyperBandScheduler() analysis = tune.run( easy_objective, metric="mean_loss", mode="min", search_alg=algo, scheduler=scheduler, num_samples=10 if smoke_test else 100, ) print("Best hyperparameters found were: ", analysis.best_config)
def testOptuna(self): from ray.tune.suggest.optuna import OptunaSearch from optuna.samplers import RandomSampler np.random.seed(1000) # At least one nan, inf, -inf and float out = tune.run( _invalid_objective, search_alg=OptunaSearch(sampler=RandomSampler(seed=1234)), config=self.config, mode="max", num_samples=8, reuse_actors=False) best_trial = out.best_trial self.assertLessEqual(best_trial.config["report"], 2.0)
def run_optuna_tune(smoke_test=False): algo = OptunaSearch(metric=["loss", "gain"], mode=["min", "max"]) algo = ConcurrencyLimiter(algo, max_concurrent=4) analysis = tune.run( easy_objective, search_alg=algo, num_samples=10 if smoke_test else 100, config={ "steps": 100, "width": tune.uniform(0, 20), "height": tune.uniform(-100, 100), # This is an ignored parameter. "activation": tune.choice(["relu", "tanh"]) }) print("Best hyperparameters for loss found were: ", analysis.get_best_config("loss", "min")) print("Best hyperparameters for gain found were: ", analysis.get_best_config("gain", "max"))
def init_search_algorithm(search_alg, metric=None, mode=None): """Specify a search algorithm and you must pip install it first. If no search algorithm is specified, the default search algorithm is BasicVariantGenerator. See more details here: https://docs.ray.io/en/master/tune/api_docs/suggestion.html Args: search_alg (str): One of 'basic_variant', 'bayesopt', or 'optuna'. metric (str): The metric to monitor for early stopping. mode (str): One of 'min' or 'max' to determine whether to minimize or maximize the metric. """ if search_alg == 'optuna': assert metric and mode, "Metric and mode cannot be None for optuna." from ray.tune.suggest.optuna import OptunaSearch return OptunaSearch(metric=metric, mode=mode) elif search_alg == 'bayesopt': assert metric and mode, "Metric and mode cannot be None for bayesian optimization." from ray.tune.suggest.bayesopt import BayesOptSearch return BayesOptSearch(metric=metric, mode=mode) logging.info(f'{search_alg} search is found, run BasicVariantGenerator().')
def run_optuna_tune(smoke_test=False): algo = OptunaSearch() algo = ConcurrencyLimiter(algo, max_concurrent=4) scheduler = AsyncHyperBandScheduler() analysis = tune.run( easy_objective, metric="mean_loss", mode="min", search_alg=algo, scheduler=scheduler, num_samples=10 if smoke_test else 100, config={ "steps": 100, "width": tune.uniform(0, 20), "height": tune.uniform(-100, 100), # This is an ignored parameter. "activation": tune.choice(["relu", "tanh"]) }) print("Best hyperparameters found were: ", analysis.best_config)
def testOptuna(self): from ray.tune.suggest.optuna import OptunaSearch from optuna.samplers import RandomSampler np.random.seed(1000) out = tune.run(_multi_objective, search_alg=OptunaSearch( sampler=RandomSampler(seed=1234), metric=["a", "b", "c"], mode=["max", "min", "max"], ), config=self.config, num_samples=16, reuse_actors=False) best_trial_a = out.get_best_trial("a", "max") self.assertGreaterEqual(best_trial_a.config["a"], 0.8) best_trial_b = out.get_best_trial("b", "min") self.assertGreaterEqual(best_trial_b.config["b"], 0.8) best_trial_c = out.get_best_trial("c", "max") self.assertGreaterEqual(best_trial_c.config["c"], 0.8)
def _test_roberta(method='BlendSearch'): max_num_epoch = 100 num_samples = -1 time_budget_s = 3600 search_space = { # You can mix constants with search space objects. "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch), "learning_rate": flaml.tune.loguniform(1e-5, 3e-5), "weight_decay": flaml.tune.uniform(0, 0.3), "per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]), "seed": flaml.tune.choice([12, 22, 33, 42]), } start_time = time.time() ray.init(num_cpus=4, num_gpus=4) if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import tuneBOHB algo = tuneBOHB(max_concurrent=4) scheduler = HyperBandForBOHB(max_t=max_num_epoch) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "num_train_epochs": 1, "per_device_train_batch_size": 128, }]) elif 'BlendSearch' == method: from flaml import BlendSearch algo = BlendSearch( points_to_evaluate=[{ "num_train_epochs": 1, "per_device_train_batch_size": 128, }]) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch(max_concurrent=3) elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1) scheduler = None analysis = ray.tune.run(train_roberta, metric=HP_METRIC, mode=MODE, resources_per_trial={ "gpu": 4, "cpu": 4 }, config=search_space, local_dir='logs/', num_samples=num_samples, time_budget_s=time_budget_s, keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC, scheduler=scheduler, search_alg=algo) ray.shutdown() best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all") metric = best_trial.metric_analysis[HP_METRIC][MODE] logger.info(f"method={method}") logger.info(f"n_trials={len(analysis.trials)}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def _test_distillbert(method='BlendSearch'): max_num_epoch = 64 num_samples = -1 time_budget_s = 10800 search_space = { # You can mix constants with search space objects. "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch), "learning_rate": flaml.tune.loguniform(1e-6, 1e-4), "adam_beta1": flaml.tune.uniform(0.8, 0.99), "adam_beta2": flaml.tune.loguniform(98e-2, 9999e-4), "adam_epsilon": flaml.tune.loguniform(1e-9, 1e-7), } start_time = time.time() ray.init(num_cpus=4, num_gpus=4) if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import tuneBOHB algo = tuneBOHB(max_concurrent=4) scheduler = HyperBandForBOHB(max_t=max_num_epoch) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "num_train_epochs": 1, }]) elif 'BlendSearch' == method: from flaml import BlendSearch algo = BlendSearch(points_to_evaluate=[{ "num_train_epochs": 1, }]) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1) scheduler = None analysis = ray.tune.run( train_distilbert, metric=HP_METRIC, mode=MODE, # You can add "gpu": 1 to allocate GPUs resources_per_trial={"gpu": 1}, config=search_space, local_dir='test/logs/', num_samples=num_samples, time_budget_s=time_budget_s, keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC, scheduler=scheduler, search_alg=algo) ray.shutdown() best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all") metric = best_trial.metric_analysis[HP_METRIC][MODE] logger.info(f"method={method}") logger.info(f"n_trials={len(analysis.trials)}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def _test_xgboost(method='BlendSearch'): try: import ray except ImportError: return if method == 'BlendSearch': from flaml import tune else: from ray import tune search_space = { # You can mix constants with search space objects. "max_depth": tune.randint(1, 8) if method in [ "BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1) } max_iter = 10 for num_samples in [256]: time_budget_s = 60 #None for n_cpu in [8]: start_time = time.time() ray.init(num_cpus=n_cpu, num_gpus=0) if method == 'BlendSearch': analysis = tune.run( train_breast_cancer, init_config={ "max_depth": 1, "min_child_weight": 3, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, report_intermediate_result=True, # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, use_ray=True) else: if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "max_depth": 1, "min_child_weight": 3, }], cat_hp_cost={ "min_child_weight": [6, 3, 2], }) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples*n_cpu) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler( max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo) ray.shutdown() # # Load the best model checkpoint # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss","min","all") accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def cifar10_main(method='BlendSearch', num_samples=10, max_num_epochs=100, gpus_per_trial=2): data_dir = os.path.abspath("test/data") load_data(data_dir) # Download data for all trials before starting the run if method == 'BlendSearch': from flaml import tune else: from ray import tune if method in ['BlendSearch', 'BOHB', 'Optuna']: config = { "l1": tune.randint(2, 8), "l2": tune.randint(2, 8), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs, q=1), "batch_size": tune.randint(1, 4) #tune.choice([2, 4, 8, 16]) } else: config = { "l1": tune.randint(2, 9), "l2": tune.randint(2, 9), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1), "batch_size": tune.randint(1, 5) #tune.choice([2, 4, 8, 16]) } import ray time_budget_s = 3600 start_time = time.time() if method == 'BlendSearch': result = tune.run(ray.tune.with_parameters(train_cifar, data_dir=data_dir), init_config={ "l1": 2, "l2": 2, "num_epochs": 1, "batch_size": 4, }, metric="loss", mode="min", max_resource=max_num_epochs, min_resource=1, report_intermediate_result=True, resources_per_trial={ "cpu": 2, "gpu": gpus_per_trial }, config=config, local_dir='logs/', num_samples=num_samples, time_budget_s=time_budget_s, use_ray=True) else: if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB() scheduler = HyperBandForBOHB(max_t=max_num_epochs) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "l1": 2, "l2": 2, "num_epochs": 1, "batch_size": 4, }]) elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1) result = tune.run(tune.with_parameters(train_cifar, data_dir=data_dir), resources_per_trial={ "cpu": 2, "gpu": gpus_per_trial }, config=config, metric="loss", mode="min", num_samples=num_samples, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo) ray.shutdown() logger.info(f"method={method}") logger.info(f"n_samples={num_samples}") logger.info(f"time={time.time()-start_time}") best_trial = result.get_best_trial("loss", "min", "all") logger.info("Best trial config: {}".format(best_trial.config)) logger.info("Best trial final validation loss: {}".format( best_trial.metric_analysis["loss"]["min"])) logger.info("Best trial final validation accuracy: {}".format( best_trial.metric_analysis["accuracy"]["max"])) best_trained_model = Net(2**best_trial.config["l1"], 2**best_trial.config["l2"]) device = "cpu" if torch.cuda.is_available(): device = "cuda:0" if gpus_per_trial > 1: best_trained_model = nn.DataParallel(best_trained_model) best_trained_model.to(device) checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint") model_state, optimizer_state = torch.load(checkpoint_path) best_trained_model.load_state_dict(model_state) test_acc = _test_accuracy(best_trained_model, device) logger.info("Best trial test set accuracy: {}".format(test_acc))
def _tune_run(self, config, resources_per_trial): """Wrapper to call ``tune.run``. Multiple estimators are generated when early stopping is possible, whereas a single estimator is generated when early stopping is not possible. Args: config (dict): Configurations such as hyperparameters to run ``tune.run`` on. resources_per_trial (dict): Resources to use per trial within Ray. Accepted keys are `cpu`, `gpu` and custom resources, and values are integers specifying the number of each resource to use. Returns: analysis (`ExperimentAnalysis`): Object returned by `tune.run`. """ if self.seed is not None: random.seed(self.seed) np.random.seed(self.seed) trainable = _Trainable if self.pipeline_auto_early_stop and check_is_pipeline( self.estimator) and self.early_stopping: trainable = _PipelineTrainable max_iter = self.max_iters if self.early_stopping is not None: config["estimator_list"] = [ clone(self.estimator) for _ in range(self.n_splits) ] if hasattr(self.early_stopping, "_max_t_attr"): # we want to delegate stopping to schedulers which # support it, but we want it to stop eventually, just in case # the solution is to make the stop condition very big max_iter = self.max_iters * 10 else: config["estimator_list"] = [self.estimator] stopper = MaximumIterationStopper(max_iter=max_iter) if self.stopper: stopper = CombinedStopper(stopper, self.stopper) run_args = dict(scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop=stopper, num_samples=self.n_trials, config=config, fail_fast="raise", resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir), loggers=self.loggers, time_budget_s=self.time_budget_s) if self.search_optimization == "random": if isinstance(self.param_distributions, list): search_algo = RandomListSearcher(self.param_distributions) else: search_algo = BasicVariantGenerator() run_args["search_alg"] = search_algo else: search_space = None override_search_space = True if self._is_param_distributions_all_tune_domains(): run_args["config"].update(self.param_distributions) override_search_space = False search_kwargs = self.search_kwargs.copy() search_kwargs.update(metric=self._metric_name, mode="max") if self.search_optimization == "bayesian": from ray.tune.suggest.skopt import SkOptSearch if override_search_space: search_space = self.param_distributions search_algo = SkOptSearch(space=search_space, **search_kwargs) run_args["search_alg"] = search_algo elif self.search_optimization == "bohb": from ray.tune.suggest.bohb import TuneBOHB if override_search_space: search_space = self._get_bohb_config_space() if self.seed: warnings.warn("'seed' is not implemented for BOHB.") search_algo = TuneBOHB(space=search_space, **search_kwargs) # search_algo = TuneBOHB( # space=search_space, seed=self.seed, **search_kwargs) run_args["search_alg"] = search_algo elif self.search_optimization == "optuna": from ray.tune.suggest.optuna import OptunaSearch from optuna.samplers import TPESampler sampler = TPESampler(seed=self.seed) if override_search_space: search_space = self._get_optuna_params() search_algo = OptunaSearch(space=search_space, sampler=sampler, **search_kwargs) run_args["search_alg"] = search_algo elif self.search_optimization == "hyperopt": from ray.tune.suggest.hyperopt import HyperOptSearch if override_search_space: search_space = self._get_hyperopt_params() search_algo = HyperOptSearch(space=search_space, random_state_seed=self.seed, **search_kwargs) run_args["search_alg"] = search_algo else: # This should not happen as we validate the input before # this method. Still, just to be sure, raise an error here. raise ValueError( f"Invalid search optimizer: {self.search_optimization}") if isinstance(self.n_jobs, int) and self.n_jobs > 0 \ and not self.search_optimization == "random": search_algo = ConcurrencyLimiter(search_algo, max_concurrent=self.n_jobs) run_args["search_alg"] = search_algo with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="fail_fast='raise' " "detected.") analysis = tune.run(trainable, **run_args) return analysis
My_config.train_dataset_path = parser.parse_args().train_data_path My_config.validation_dataset_path = parser.parse_args().valid_data_path My_config.model = parser.parse_args().model My_config.train_batchsize for class_name in os.listdir(My_config.train_dataset_path): My_config.FolderNames2English_names[str( My_config.num_class)] = class_name My_config.num_class += 1 My_config.class_num = My_config.num_class gpus_per_trial = 1 sample_num = parser.parse_args().sample_num dim = random_dim_by_CompoundMethod(parser.parse_args().maxrange) algo = OptunaSearch(sampler=TPESampler()) scheduler = AsyncHyperBandScheduler() search_space = {"dim": tune.choice(dim[0:len(dim)])} analysis = run(Train_Net, name="Model_Scaling", config=search_space, num_samples=10, metric="accuracy", mode="max", search_alg=algo, scheduler=scheduler, resources_per_trial={ "cpu": 48, "gpu": gpus_per_trial
def run_tuning( config: DictConfig, output_config_file: str, train_fn: Callable[[Dict[str, Any], Any, Any], None], config_fn: Callable[[Dict[str, Any]], Dict[str, Any]], train: Any = None, val: Any = None, ): """Run distributed hyperparameter tuning using ray tune Uses Optuna TPE search algorithm and ASHA pruning strategy Args: config (omegaconf.DictConfig): The parsed configuration output_config_file (str): Path to save the optimal configuration that yields the best result train_fn (Callable[[Dict[str, Any], Any, Any], None]): Train function that takes the configuration as a python dict, train dataset and validation dataset and fits the model. This function is used to create the trainable that will run when calling ray.tune.run config_fn (Callable[[Dict[str, Any]], Dict[str, Any]]): Configuration function that constructs the search space by overriding entries in the input configuration train (Dataset): Torch dataset or corpus that will be used for training val (Dataset): Torch dataset or corpus that will be used for validation Returns: Dict[str, Any]: The configuration for the best trial Examples: >>> # Make search space >>> def configure_search_space(config): >>> config["optimizer"] = tune.choice(["SGD", "Adam", "AdamW"]) >>> config["optim"]["lr"] = tune.loguniform(1e-4, 1e-1) >>> config["optim"]["weight_decay"] = tune.loguniform(1e-4, 1e-1) >>> config["data"]["batch_size"] = tune.choice([16, 32, 64, 128]) >>> return config >>> # Training function. >>> def train_fn(config, train=None, val=None): >>> config = OmegaConf.create(config) # convert dict from ray tune to DictConfig >>> ldm = PLDataModuleFromDatasets(train, val=val, seed=config.seed, no_test_set=True, **config.data) >>> model = Net(**config.model) >>> optimizer = getattr(optim, config.optimizer)(model.parameters(), **config.optim) >>> criterion = nn.CrossEntropyLoss() >>> lm = PLModule( >>> model, optimizer, criterion, >>> hparams=config, >>> metrics={"acc": FromLogits(pl.metrics.classification.Accuracy())}, # Logs train_acc and val_acc >>> ) >>> metrics_map = {"accuracy": "val_acc", "validation_loss": "val_loss"} # map metrics from pl to ray tune >>> trainer = make_trainer_for_ray_tune(metrics_map=metrics_map, **config.trainer) >>> trainer.fit(lm, datamodule=ldm) >>> # Run optimization >>> if __name__ == "__main__": >>> config, train_dataset, val_dataset = ... >>> best_config = run_tuning( >>> config, >>> "configs/best.tuning.config.yml", >>> train_fn, >>> configure_search_space, >>> train_dataset, >>> val_dataset, >>> ) """ config = _extract_wandb_config(config) cfg = config_fn(cast(Dict[str, Any], OmegaConf.to_container(config))) cfg["trainer"]["gpus"] = math.ceil(cfg["tune"]["gpus_per_trial"]) trainable = tune.with_parameters(train_fn, train=train, val=val) metric, mode = cfg["tune"]["metric"], cfg["tune"]["mode"] analysis = tune.run( trainable, loggers=[ WandbLogger ], # WandbLogger logs experiment configurations and metrics reported via tune.report() to W&B Dashboard resources_per_trial={ "cpu": cfg["tune"]["cpus_per_trial"], "gpu": cfg["tune"]["gpus_per_trial"], }, config=cfg, max_failures=10, num_samples=cfg["tune"]["num_trials"], search_alg=OptunaSearch(metric=metric, mode=mode), metric=metric, mode=mode, # scheduler=tune.schedulers.ASHAScheduler(metric=metric, mode=mode, reduction_factor=2), name=f"{cfg['trainer']['experiment_name']}-tuning", ) best_config = analysis.get_best_config(metric, mode) best_result = analysis.get_best_trial(metric=metric, mode=mode).last_result logger.info(f"Best hyperparameters found were: {best_config}") logger.info(f"Best score: {best_result[metric]}") best_config["tune"]["result"] = best_result yaml_dump(best_config, output_config_file) return best_config
def _tune_run(self, config, resources_per_trial): """Wrapper to call ``tune.run``. Multiple estimators are generated when early stopping is possible, whereas a single estimator is generated when early stopping is not possible. Args: config (dict): Configurations such as hyperparameters to run ``tune.run`` on. resources_per_trial (dict): Resources to use per trial within Ray. Accepted keys are `cpu`, `gpu` and custom resources, and values are integers specifying the number of each resource to use. Returns: analysis (`ExperimentAnalysis`): Object returned by `tune.run`. """ stop_condition = {"training_iteration": self.max_iters} if self.early_stopping is not None: config["estimator_list"] = [ clone(self.estimator) for _ in range(self.n_splits) ] if hasattr(self.early_stopping, "_max_t_attr"): # we want to delegate stopping to schedulers which # support it, but we want it to stop eventually, just in case # the solution is to make the stop condition very big stop_condition = {"training_iteration": self.max_iters * 10} else: config["estimator_list"] = [self.estimator] if self.search_optimization == "random": run_args = dict(scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop=stop_condition, num_samples=self.num_samples, config=config, fail_fast=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) if isinstance(self.param_distributions, list): run_args["search_alg"] = RandomListSearcher( self.param_distributions) analysis = tune.run(_Trainable, **run_args) return analysis elif self.search_optimization == "bayesian": from skopt import Optimizer from ray.tune.suggest.skopt import SkOptSearch hyperparameter_names, spaces = self._get_skopt_params() search_algo = SkOptSearch(Optimizer(spaces), hyperparameter_names, metric="average_test_score", **self.search_kwargs) elif self.search_optimization == "bohb": from ray.tune.suggest.bohb import TuneBOHB config_space = self._get_bohb_config_space() search_algo = TuneBOHB(config_space, metric="average_test_score", mode="max", **self.search_kwargs) elif self.search_optimization == "optuna": from ray.tune.suggest.optuna import OptunaSearch config_space = self._get_optuna_params() search_algo = OptunaSearch(config_space, metric="average_test_score", mode="max", **self.search_kwargs) elif self.search_optimization == "hyperopt": from ray.tune.suggest.hyperopt import HyperOptSearch config_space = self._get_hyperopt_params() search_algo = HyperOptSearch(config_space, metric="average_test_score", mode="max", **self.search_kwargs) if isinstance(self.n_jobs, int) and self.n_jobs > 0: search_algo = ConcurrencyLimiter(search_algo, max_concurrent=self.n_jobs) analysis = tune.run(_Trainable, search_alg=search_algo, scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop=stop_condition, num_samples=self.num_samples, config=config, fail_fast=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) return analysis
# Feed the score back back to Tune. tune.report(iterations=step, mean_loss=intermediate_score) time.sleep(0.1) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument( "--smoke-test", action="store_true", help="Finish quickly for testing") args, _ = parser.parse_known_args() ray.init(configure_logging=False) space = [ param.suggest_uniform("width", 0, 20), param.suggest_uniform("height", -100, 100), # This is an ignored parameter. param.suggest_categorical("activation", ["relu", "tanh"]) ] config = { "num_samples": 10 if args.smoke_test else 100, "config": { "steps": 100, } } algo = OptunaSearch(space, metric="mean_loss", mode="min") scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min") tune.run(easy_objective, search_alg=algo, scheduler=scheduler, **config)
def testConvertOptuna(self): from ray.tune.suggest.optuna import OptunaSearch, param import optuna from optuna.samplers import RandomSampler # Grid search not supported, should raise ValueError with self.assertRaises(ValueError): OptunaSearch.convert_search_space( {"grid": tune.grid_search([0, 1])}) config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = OptunaSearch.convert_search_space(config) optuna_config = { "a": optuna.distributions.CategoricalDistribution([2, 3, 4]), "b": { "x": optuna.distributions.IntUniformDistribution(0, 5, step=2), "z": optuna.distributions.LogUniformDistribution(1e-4, 1e-2) } } legacy_optuna_config = [ param.suggest_categorical("a", [2, 3, 4]), param.suggest_int("b/x", 0, 5, 2), param.suggest_loguniform("b/z", 1e-4, 1e-2) ] sampler1 = RandomSampler(seed=1234) searcher1 = OptunaSearch(space=converted_config, sampler=sampler1, metric="a", mode="max") sampler2 = RandomSampler(seed=1234) searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2, metric="a", mode="max") sampler3 = RandomSampler(seed=1234) searcher3 = OptunaSearch(space=legacy_optuna_config, sampler=sampler3, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") config3 = searcher3.suggest("0") self.assertEqual(config1, config2) self.assertEqual(config1, config3) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = OptunaSearch(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4] mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix List and Dict } searcher = OptunaSearch(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)
def main(args): model = args.model config_file = args.config_file log_dir = Path(f"./logs/{model.value}") log_dir.mkdir(exist_ok=True, parents=True) data_root = Path(f"./data/processed/{model.value}/IFCNetCore").absolute() with open("IFCNetCore_Classes.json", "r") as f: class_names = json.load(f) if model == Model.MVCNN: config = { "batch_size": 64, "learning_rate": tune.loguniform(1e-5, 1e-2), "weight_decay": tune.loguniform(1e-4, 1e-2), "cnn_name": tune.choice(["vgg11", "resnet34", "resnet50"]), "pretrained": True, "epochs": 30, "num_views": 12 } train_func = partial(train_mvcnn, data_root=data_root, class_names=class_names, eval_on_test=config_file is not None) elif model == Model.DGCNN: config = { "batch_size": 8, "learning_rate": tune.loguniform(1e-4, 1e-2), "weight_decay": tune.loguniform(1e-4, 1e-2), "k": tune.choice([20, 30, 40]), "embedding_dim": tune.choice([516, 1024, 2048]), "dropout": 0.5, "epochs": 250 } train_func = partial(train_dgcnn, data_root=data_root, class_names=class_names, eval_on_test=config_file is not None) elif model == Model.MeshNet: config = { "batch_size": tune.choice([8, 16, 32]), "learning_rate": tune.loguniform(1e-4, 1e-2), "weight_decay": tune.loguniform(1e-4, 1e-2), "num_kernel": tune.choice([32, 64]), "sigma": tune.choice([0.1, 0.2, 0.3]), "aggregation_method": tune.choice(["Concat", "Max", "Average"]), "epochs": 150 } train_func = partial(train_meshnet, data_root=data_root, class_names=class_names, eval_on_test=config_file is not None) if config_file: with config_file.open("r") as f: config = json.load(f) scheduler = ASHAScheduler(max_t=250, grace_period=10) reporter = CLIReporter(metric_columns=[ "train_balanced_accuracy_score", "val_balanced_accuracy_score", "training_iteration" ]) result = tune.run( train_func, resources_per_trial={ "cpu": 8, "gpu": 1 }, local_dir=log_dir, config=config, mode="max", metric="val_balanced_accuracy_score", search_alg=OptunaSearch() if config_file is None else None, num_samples=20 if config_file is None else 1, scheduler=scheduler if config_file is None else None, progress_reporter=reporter) best_trial = result.get_best_trial("val_balanced_accuracy_score", "max", "last") print("Best trial config: {}".format(best_trial.config)) print("Best trial final validation accuracy (balanced): {}".format( best_trial.last_result["val_balanced_accuracy_score"]))
# Feed the score back back to Tune. tune.report(iterations=step, mean_loss=intermediate_score) time.sleep(0.1) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument("--smoke-test", action="store_true", help="Finish quickly for testing") args, _ = parser.parse_known_args() ray.init(configure_logging=False) algo = OptunaSearch() algo = ConcurrencyLimiter(algo, max_concurrent=4) scheduler = AsyncHyperBandScheduler() analysis = tune.run( easy_objective, metric="mean_loss", mode="min", search_alg=algo, scheduler=scheduler, num_samples=10 if args.smoke_test else 100, config={ "steps": 100, "width": tune.uniform(0, 20), "height": tune.uniform(-100, 100), # This is an ignored parameter. "activation": tune.choice(["relu", "tanh"])
def _test_xgboost(method="BlendSearch"): try: import ray except ImportError: return if method == "BlendSearch": from flaml import tune else: from ray import tune search_space = { "max_depth": tune.randint(1, 9) if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1), } max_iter = 10 for num_samples in [128]: time_budget_s = 60 for n_cpu in [2]: start_time = time.time() # ray.init(address='auto') if method == "BlendSearch": analysis = tune.run( train_breast_cancer, config=search_space, low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, scheduler="asha", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, use_ray=True, ) else: if "ASHA" == method: algo = None elif "BOHB" == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif "Optuna" == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif "CFO" == method: from flaml import CFO algo = CFO( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "CFOCat" == method: from flaml.searcher.cfo_cat import CFOCat algo = CFOCat( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "Dragonfly" == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif "SkOpt" == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif "Nevergrad" == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif "ZOOpt" == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples * n_cpu) elif "Ax" == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif "HyperOpt" == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != "BOHB": from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo, ) # # Load the best model checkpoint # import os # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss", "min", "all") accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")