def testSkopt(self): from ray.tune.suggest.skopt import SkOptSearch searcher = SkOptSearch(space=self.config, metric=self.metric_name, mode="max") self._save(searcher) searcher = SkOptSearch(space=self.config, metric=self.metric_name, mode="max") self._restore(searcher)
def testConvertSkOpt(self): from ray.tune.suggest.skopt import SkOptSearch config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = SkOptSearch.convert_search_space(config) skopt_config = {"a": [2, 3, 4], "b/x": (0, 5), "b/z": (1e-4, 1e-2)} searcher1 = SkOptSearch(space=converted_config) searcher2 = SkOptSearch(space=skopt_config) np.random.seed(1234) config1 = searcher1.suggest("0") np.random.seed(1234) config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = SkOptSearch(metric="a", mode="max") analysis = tune.run( _mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] self.assertIn(trial.config["a"], [2, 3, 4]) self.assertEqual(trial.config["b"]["y"], 4)
def setup_tune_scheduler(): from ray.tune.suggest.skopt import SkOptSearch from ray.tune.suggest.suggestion import ConcurrencyLimiter from skopt import Optimizer exp_metrics = workload.exp_metric() search_space, dim_names = workload.create_skopt_space() algo = ConcurrencyLimiter( SkOptSearch( Optimizer(search_space), dim_names, **exp_metrics, ), 81, ) scheduler = FluidBandScheduler( max_res=81, reduction_factor=3, **exp_metrics, ) return dict( search_alg=algo, scheduler=scheduler, trial_executor=MyRayTrialExecutor(), resources_per_trial=com.detect_baseline_resource(), )
def run_batch(s): f = open( "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_args", "rb") args = pickle.load(f) f = open( "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_spaces", "rb") hyperspaces = pickle.load(f) for i in s: current_space = hyperspaces[i] optimizer = Optimizer(current_space) if args.model == "segmentation_cityscapes" or args.model == "segmentation_gis": search_algo = SkOptSearch( optimizer, ['learning_rate', 'epochs', 'batch_size', 'adam_epsilon'], metric='average_res', mode='max') else: search_algo = SkOptSearch(optimizer, [ 'learning_rate', 'dropout', 'epochs', 'batch_size', 'adam_epsilon' ], metric='average_res', mode='max') analysis = tune.run( multi_train, search_alg=search_algo, num_samples=int(args.trials), resources_per_trial={ 'cpu': 25, 'gpu': 1 }, local_dir="/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/ray_results" ) df = analysis.results_df df_name = "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/" + args.out + "/" df_name += "space_" df_name += str(i) df_name += ".csv" df.to_csv(df_name) print("Finished space " + args.space) print( "Finished all spaces. Files writtten to /lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/" + args.out)
def testConvergenceSkOpt(self): from ray.tune.suggest.skopt import SkOptSearch np.random.seed(0) searcher = SkOptSearch() analysis = self._testConvergence(searcher) assert len(analysis.trials) < 100 assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-3)
def testSkopt(self): from ray.tune.suggest.skopt import SkOptSearch np.random.seed(1234) # At least one nan, inf, -inf and float out = tune.run(_invalid_objective, search_alg=SkOptSearch(), config=self.config, mode="max", num_samples=8, reuse_actors=False) best_trial = out.best_trial self.assertLessEqual(best_trial.config["report"], 2.0)
def set_basic_conf(self): optimizer = skopt.Optimizer([(0, 20), (-100, 100)]) previously_run_params = [[10, 0], [15, -20]] known_rewards = [-189, -1144] def cost(space, reporter): reporter(loss=(space["height"]**2 + space["width"]**2)) search_alg = SkOptSearch(optimizer, ["width", "height"], metric="loss", mode="min", points_to_evaluate=previously_run_params, evaluated_rewards=known_rewards) return search_alg, cost
def testSkOpt(self): from ray.tune.suggest.skopt import SkOptSearch searcher = SkOptSearch( space=self.space, metric="metric", mode="max", ) point = { self.param_name: self.valid_value, } get_len_X = lambda s: len(s._skopt_opt.Xi) # noqa E731 get_len_y = lambda s: len(s._skopt_opt.yi) # noqa E731 self.run_add_evaluated_point(point, searcher, get_len_X, get_len_y) self.run_add_evaluated_trials(searcher, get_len_X, get_len_y)
def fit(self, metric="log_test_fold_pvalue", num_samples=10, iterations=1, max_concurrent=4, distribute_deepprog=False, timesteps_total=100): """ """ self._distribute_deepprog = distribute_deepprog config = { "num_samples": num_samples, "config": { "iterations": iterations, }, "stop": { "timesteps_total": timesteps_total }, } metric_authorized = { "log_test_fold_pvalue": "max", "test_fold_cindex": "max", "cluster_consistency": "max", "log_full_pvalue": "max", "sum_log_pval": "max", "log_test_pval": "max", "test_cindex": "max", "mix_score": "max", "test_consisentcy": "max", } try: assert metric in metric_authorized except Exception: raise (Exception('{0} should be in {1}'.format( metric, metric_authorized))) optimizer_header = self.args_to_optimize.keys() optimizer_value = [ self.args_to_optimize[key] for key in optimizer_header ] # optimizer_value += [tuple(norm.items()) for norm in self.normalization] optimizer = Optimizer(optimizer_value) algo = SkOptSearch( optimizer, list(optimizer_header), max_concurrent=max_concurrent, metric=metric, mode=metric_authorized[metric], ) scheduler = AsyncHyperBandScheduler(metric=metric, mode=metric_authorized[metric]) self.results = run(self._objective_only_training, name=self.project_name, search_alg=algo, scheduler=scheduler, **config) index = ['config/' + key for key in self.args_to_optimize] index = ['trial_name'] + index + \ ["test_pval_{0}".format(key) for key in self.test_datasets] + [metric, "full_pvalue"] df = self.results.dataframe()[index] print('#### best results obtained with:\n{0}'.format( tabulate(df, headers='keys', tablefmt='psql'))) fname = '{0}/{1}_hyperparameter_scores_summary.tsv'.format( self.path_results, self.project_name) df.to_csv(fname, sep="\t") print('File :{0} written'.format(fname))
def _test_roberta(method='BlendSearch'): max_num_epoch = 100 num_samples = -1 time_budget_s = 3600 search_space = { # You can mix constants with search space objects. "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch), "learning_rate": flaml.tune.loguniform(1e-5, 3e-5), "weight_decay": flaml.tune.uniform(0, 0.3), "per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]), "seed": flaml.tune.choice([12, 22, 33, 42]), } start_time = time.time() ray.init(num_cpus=4, num_gpus=4) if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import tuneBOHB algo = tuneBOHB(max_concurrent=4) scheduler = HyperBandForBOHB(max_t=max_num_epoch) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "num_train_epochs": 1, "per_device_train_batch_size": 128, }]) elif 'BlendSearch' == method: from flaml import BlendSearch algo = BlendSearch( points_to_evaluate=[{ "num_train_epochs": 1, "per_device_train_batch_size": 128, }]) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch(max_concurrent=3) elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1) scheduler = None analysis = ray.tune.run(train_roberta, metric=HP_METRIC, mode=MODE, resources_per_trial={ "gpu": 4, "cpu": 4 }, config=search_space, local_dir='logs/', num_samples=num_samples, time_budget_s=time_budget_s, keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC, scheduler=scheduler, search_alg=algo) ray.shutdown() best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all") metric = best_trial.metric_analysis[HP_METRIC][MODE] logger.info(f"method={method}") logger.info(f"n_trials={len(analysis.trials)}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def _test_xgboost(method='BlendSearch'): try: import ray except ImportError: return if method == 'BlendSearch': from flaml import tune else: from ray import tune search_space = { # You can mix constants with search space objects. "max_depth": tune.randint(1, 8) if method in [ "BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1) } max_iter = 10 for num_samples in [256]: time_budget_s = 60 #None for n_cpu in [8]: start_time = time.time() ray.init(num_cpus=n_cpu, num_gpus=0) if method == 'BlendSearch': analysis = tune.run( train_breast_cancer, init_config={ "max_depth": 1, "min_child_weight": 3, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, report_intermediate_result=True, # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, use_ray=True) else: if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "max_depth": 1, "min_child_weight": 3, }], cat_hp_cost={ "min_child_weight": [6, 3, 2], }) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples*n_cpu) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler( max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo) ray.shutdown() # # Load the best model checkpoint # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss","min","all") accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def _tune_run(self, config, resources_per_trial): """Wrapper to call ``tune.run``. Multiple estimators are generated when early stopping is possible, whereas a single estimator is generated when early stopping is not possible. Args: config (dict): Configurations such as hyperparameters to run ``tune.run`` on. resources_per_trial (dict): Resources to use per trial within Ray. Accepted keys are `cpu`, `gpu` and custom resources, and values are integers specifying the number of each resource to use. Returns: analysis (`ExperimentAnalysis`): Object returned by `tune.run`. """ if self.early_stopping is not None: config["estimator"] = [ clone(self.estimator) for _ in range(self.n_splits) ] else: config["estimator"] = self.estimator if self.search_optimization == "random": if isinstance(self.param_distributions, list): analysis = tune.run( _Trainable, scheduler=self.early_stopping, search_alg=RandomListSearcher(self.param_distributions), reuse_actors=True, verbose=self.verbose, stop={"training_iteration": self.max_iters}, num_samples=self.num_samples, config=config, fail_fast=True, checkpoint_at_end=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) else: analysis = tune.run( _Trainable, scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop={"training_iteration": self.max_iters}, num_samples=self.num_samples, config=config, fail_fast=True, checkpoint_at_end=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) else: hyperparameter_names, spaces = self._get_skopt_params() search_algo = SkOptSearch( Optimizer(spaces), hyperparameter_names, metric="average_test_score") analysis = tune.run( _Trainable, search_alg=search_algo, scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop={"training_iteration": self.max_iters}, num_samples=self.num_samples, config=config, fail_fast=True, checkpoint_at_end=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) return analysis
config = { "num_samples": 10 if args.smoke_test else 50, "config": { "iterations": 100, }, "stop": { "timesteps_total": 100 }, } optimizer = Optimizer([(0, 20), (-100, 100)]) previously_run_params = [[10, 0], [15, -20]] known_rewards = [-189, -1144] algo = SkOptSearch(optimizer, ["width", "height"], max_concurrent=4, metric="mean_loss", mode="min", points_to_evaluate=previously_run_params, evaluated_rewards=known_rewards) scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min") run(easy_objective, name="skopt_exp_with_warmstart", search_alg=algo, scheduler=scheduler, **config) # Now run the experiment without known rewards algo = SkOptSearch(optimizer, ["width", "height"], max_concurrent=4, metric="mean_loss", mode="min",
def _tune_run(self, config, resources_per_trial): """Wrapper to call ``tune.run``. Multiple estimators are generated when early stopping is possible, whereas a single estimator is generated when early stopping is not possible. Args: config (dict): Configurations such as hyperparameters to run ``tune.run`` on. resources_per_trial (dict): Resources to use per trial within Ray. Accepted keys are `cpu`, `gpu` and custom resources, and values are integers specifying the number of each resource to use. Returns: analysis (`ExperimentAnalysis`): Object returned by `tune.run`. """ if self.seed is not None: random.seed(self.seed) np.random.seed(self.seed) trainable = _Trainable if self.pipeline_auto_early_stop and check_is_pipeline( self.estimator) and self.early_stopping: trainable = _PipelineTrainable max_iter = self.max_iters if self.early_stopping is not None: config["estimator_list"] = [ clone(self.estimator) for _ in range(self.n_splits) ] if hasattr(self.early_stopping, "_max_t_attr"): # we want to delegate stopping to schedulers which # support it, but we want it to stop eventually, just in case # the solution is to make the stop condition very big max_iter = self.max_iters * 10 else: config["estimator_list"] = [self.estimator] stopper = MaximumIterationStopper(max_iter=max_iter) if self.stopper: stopper = CombinedStopper(stopper, self.stopper) run_args = dict(scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop=stopper, num_samples=self.n_trials, config=config, fail_fast="raise", resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir), loggers=self.loggers, time_budget_s=self.time_budget_s) if self.search_optimization == "random": if isinstance(self.param_distributions, list): search_algo = RandomListSearcher(self.param_distributions) else: search_algo = BasicVariantGenerator() run_args["search_alg"] = search_algo else: search_space = None override_search_space = True if self._is_param_distributions_all_tune_domains(): run_args["config"].update(self.param_distributions) override_search_space = False search_kwargs = self.search_kwargs.copy() search_kwargs.update(metric=self._metric_name, mode="max") if self.search_optimization == "bayesian": from ray.tune.suggest.skopt import SkOptSearch if override_search_space: search_space = self.param_distributions search_algo = SkOptSearch(space=search_space, **search_kwargs) run_args["search_alg"] = search_algo elif self.search_optimization == "bohb": from ray.tune.suggest.bohb import TuneBOHB if override_search_space: search_space = self._get_bohb_config_space() if self.seed: warnings.warn("'seed' is not implemented for BOHB.") search_algo = TuneBOHB(space=search_space, **search_kwargs) # search_algo = TuneBOHB( # space=search_space, seed=self.seed, **search_kwargs) run_args["search_alg"] = search_algo elif self.search_optimization == "optuna": from ray.tune.suggest.optuna import OptunaSearch from optuna.samplers import TPESampler sampler = TPESampler(seed=self.seed) if override_search_space: search_space = self._get_optuna_params() search_algo = OptunaSearch(space=search_space, sampler=sampler, **search_kwargs) run_args["search_alg"] = search_algo elif self.search_optimization == "hyperopt": from ray.tune.suggest.hyperopt import HyperOptSearch if override_search_space: search_space = self._get_hyperopt_params() search_algo = HyperOptSearch(space=search_space, random_state_seed=self.seed, **search_kwargs) run_args["search_alg"] = search_algo else: # This should not happen as we validate the input before # this method. Still, just to be sure, raise an error here. raise ValueError( f"Invalid search optimizer: {self.search_optimization}") if isinstance(self.n_jobs, int) and self.n_jobs > 0 \ and not self.search_optimization == "random": search_algo = ConcurrencyLimiter(search_algo, max_concurrent=self.n_jobs) run_args["search_alg"] = search_algo with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="fail_fast='raise' " "detected.") analysis = tune.run(trainable, **run_args) return analysis
def main(args): # 1. load config print('Importing architecture from %s' % args.arch_module) arch_mod = import_module(args.arch_module) prob_mods = [] for prob_module_path in args.prob_modules: print('Importing problem from %s' % prob_module_path) this_prob_mod = import_module(prob_module_path) prob_mods.append(this_prob_mod) # 2. spool up Ray new_cluster = args.ray_connect is None ray_kwargs = {} if not new_cluster: ray_kwargs["redis_address"] = args.ray_connect assert args.ray_ncpus is None, \ "can't provide --ray-ncpus and --ray-connect" else: if args.ray_ncpus is not None: assert args.job_ncpus is None \ or args.job_ncpus <= args.ray_ncpus, \ "must have --job-ncpus <= --ray-ncpus if both given" ray_kwargs["num_cpus"] = args.ray_ncpus ray.init(**ray_kwargs) max_par_trials = args.max_par_trials if max_par_trials is None: # leave some room for hyperthread-caused over-counting of CPUs (a /2 # factor), and for running eval trials in parallel max_par_trials = max(1, multiprocessing.cpu_count() // 5) sk_space = OrderedDict() # originally I had this split between 2/3, but I think 3 is a bit too slow # on some problems, so I want to stick to 2 (even though exbw really seems # to benefit from 3) sk_space['num_layers'] = [2] sk_space['hidden_size'] = (12, 20) # empty list; no steps down, just a single fixed learning rate sk_space['learning_rate_steps'] = [()] sk_space['supervised_learning_rate'] = (1e-4, 1e-2, 'log-uniform') # these ranges are similar to my original config, which seemed to work okay sk_space['supervised_batch_size'] = (48, 128) sk_space['opt_batch_per_epoch'] = (300, 1200) # (150, 1500) # we use categorical vars to add "switched off entirely" as options (as # opposed to just "turned down very low"); I suspect switching off entirely # is good for some of those things sk_space['dropout'] = [0, 0.1, 0.25] sk_space['l1_reg'] = [0.0] # (1e-10, 1e-2, 'log-uniform') sk_space['l2_reg'] = (1e-5, 1e-2, 'log-uniform') sk_space['target_rollouts_per_epoch'] = (30, 150) if arch_mod.TEACHER_PLANNER == 'ssipp': # only relevant for SSiPP # (originally I had both h-add and lm-cut as options, but lm-cut didn't # seem to help much, so I'm leaving it out) sk_space['ssipp_teacher_heuristic'] = ['h-add'] # using random forest b/c we have lots of discrete params, & a few # categorical sk_optimiser = Optimizer(list(sk_space.values()), base_estimator='RF') algo = SkOptSearch( sk_optimiser, sk_space.keys(), max_concurrent=max_par_trials, metric='coverage', mode='max') perform_trial = make_perform_trial(arch_mod, prob_mods) tune.run( perform_trial, search_alg=algo, local_dir=args.work_dir, resources_per_trial={"cpu": 0}, num_samples=1000)
def build_search_alg(search_alg, param_ranges: dict): """ Initialize a search algorithm that is selected using 'search_alg' Parameters ---------- search_alg : str; Selecting the search algorithm. Possible values [BayesOpt, SkOpt] param_ranges : dictionary of parameter ranges over which the search should be performed Returns ------- alg : Object of the RayTune search algorithm selected """ alg = None if search_alg == "BayesOpt": from ray.tune.suggest.bayesopt import BayesOptSearch alg = BayesOptSearch( param_ranges, max_concurrent=max_concurrent, metric="test_accuracy", mode="max", utility_kwargs={ "kind": "ucb", "kappa": 2.5, "xi": 0.0 }, ) elif search_alg == "SkOpt": from skopt import Optimizer from skopt.space import Real, Integer from ray.tune.suggest.skopt import SkOptSearch opt_params = [ Integer(param_ranges["n_estimators"][0], param_ranges["n_estimators"][1]), Integer(param_ranges["max_depth"][0], param_ranges["max_depth"][1]), Real( param_ranges["max_features"][0], param_ranges["max_features"][1], prior="log-uniform", ), ] optimizer = Optimizer(opt_params) alg = SkOptSearch( optimizer, list(param_ranges.keys()), max_concurrent=max_concurrent, metric="test_accuracy", mode="max", ) else: print("Unknown Option. Select BayesOpt or SkOpt") return alg
def get_raytune_search_alg(raytune_cfg, seeds=False): if (raytune_cfg["sched"] == "pbt") or (raytune_cfg["sched"] == "pb2"): if raytune_cfg["search_alg"] is not None: print( "INFO: Using schedule '{}' is not compatible with Ray Tune search algorithms." .format(raytune_cfg["sched"])) print( "INFO: Uing the Ray Tune {} scheduler without search algorithm" .format(raytune_cfg["sched"])) return None if (raytune_cfg["sched"] == "bohb") or (raytune_cfg["sched"] == "BOHB"): print( "INFO: Using TuneBOHB search algorithm since it is required for BOHB shedule" ) if seeds: seed = 1234 else: seed = None return TuneBOHB(metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], seed=seed) # requires pip install bayesian-optimization if raytune_cfg["search_alg"] == "bayes": print("INFO: Using BayesOptSearch") return BayesOptSearch( metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], random_search_steps=raytune_cfg["bayes"]["n_random_steps"], ) # requires pip install hyperopt if raytune_cfg["search_alg"] == "hyperopt": print("INFO: Using HyperOptSearch") return HyperOptSearch( metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], n_initial_points=raytune_cfg["hyperopt"]["n_random_steps"], # points_to_evaluate=, ) if raytune_cfg["search_alg"] == "scikit": print("INFO: Using bayesian optimization from scikit-learn") return SkOptSearch( metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], convert_to_python=True, ) if raytune_cfg["search_alg"] == "nevergrad": print("INFO: Using bayesian optimization from nevergrad") return NevergradSearch( optimizer=ng.optimizers.BayesOptim( pca=False, init_budget=raytune_cfg["nevergrad"]["n_random_steps"]), metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], ) # HEBO is not yet supported # if (raytune_cfg["search_alg"] == "hebo") or (raytune_cfg["search_alg"] == "HEBO"): # print("Using HEBOSearch") # return HEBOSearch( # metric=raytune_cfg["default_metric"], # mode=raytune_cfg["default_mode"], # # max_concurrent=8, # ) else: print("INFO: Not using any Ray Tune search algorithm") return None
ncalls = 20 if TT: parameter_names = ['lr'] space = [Real(10**-7, 10**-3, "log-uniform", name='lr')] else: parameter_names = ['lr'] space = [Real(10**-7, 10**-3, "log-uniform", name='lr')] ray.init(num_cpus=nCPU, num_gpus=nGPU) optimizer = Optimizer(dimensions=space, random_state=1, base_estimator='gp') algo = SkOptSearch(optimizer, parameter_names=parameter_names, max_concurrent=4, metric="result", mode="max") scheduler = FIFOScheduler() tune.register_trainable("train_func", train) import time, random time.sleep(random.uniform(0.0, 10.0)) tune.run_experiments( { 'my_experiment': { 'run': 'train_func', 'resources_per_trial': { "cpu": int(nCPU * load // nGPU), "gpu": load },
def compile( self, input_df, model_create_func, search_space, recipe, feature_transformers=None, # model=None, future_seq_len=1, validation_df=None, mc=False, metric="mse", metric_mode="min"): """ Do necessary preparations for the engine :param input_df: :param search_space: :param num_samples: :param stop: :param search_algorithm: :param search_algorithm_params: :param fixed_params: :param feature_transformers: :param model: :param validation_df: :param metric: :return: """ # prepare parameters for search engine runtime_params = recipe.runtime_params() num_samples = runtime_params['num_samples'] stop = dict(runtime_params) search_algorithm_params = recipe.search_algorithm_params() search_algorithm = recipe.search_algorithm() fixed_params = recipe.fixed_params() schedule_algorithm = recipe.scheduler_algorithm() del stop['num_samples'] self.search_space = self._prepare_tune_config(search_space) self.stop_criteria = stop self.num_samples = num_samples if schedule_algorithm == 'AsyncHyperBand': from ray.tune.schedulers import AsyncHyperBandScheduler self.sched = AsyncHyperBandScheduler( time_attr="training_iteration", metric="reward_metric", mode="max", max_t=50, grace_period=1, reduction_factor=3, brackets=3, ) else: from ray.tune.schedulers import FIFOScheduler self.sched = FIFOScheduler() if search_algorithm == 'BayesOpt': self.search_algorithm = BayesOptSearch( self.search_space, metric="reward_metric", mode="max", utility_kwargs=search_algorithm_params["utility_kwargs"]) elif search_algorithm == 'SkOpt': from skopt import Optimizer from ray.tune.suggest.skopt import SkOptSearch opt_params = recipe.opt_params() optimizer = Optimizer(opt_params) self.search_algorithm = SkOptSearch( optimizer, list(self.search_space.keys()), metric="reward_metric", mode="max", ) else: self.search_algorithm = None self.fixed_params = fixed_params self.train_func = self._prepare_train_func( input_df=input_df, model_create_func=model_create_func, feature_transformers=feature_transformers, future_seq_len=future_seq_len, validation_df=validation_df, metric=metric, metric_mode=metric_mode, mc=mc, remote_dir=self.remote_dir)
def _test_xgboost(method="BlendSearch"): try: import ray except ImportError: return if method == "BlendSearch": from flaml import tune else: from ray import tune search_space = { "max_depth": tune.randint(1, 9) if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1), } max_iter = 10 for num_samples in [128]: time_budget_s = 60 for n_cpu in [2]: start_time = time.time() # ray.init(address='auto') if method == "BlendSearch": analysis = tune.run( train_breast_cancer, config=search_space, low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, scheduler="asha", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, use_ray=True, ) else: if "ASHA" == method: algo = None elif "BOHB" == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif "Optuna" == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif "CFO" == method: from flaml import CFO algo = CFO( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "CFOCat" == method: from flaml.searcher.cfo_cat import CFOCat algo = CFOCat( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "Dragonfly" == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif "SkOpt" == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif "Nevergrad" == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif "ZOOpt" == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples * n_cpu) elif "Ax" == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif "HyperOpt" == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != "BOHB": from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo, ) # # Load the best model checkpoint # import os # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss", "min", "all") accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def hparams(algorithm, scheduler, num_samples, tensorboard, bare): from glob import glob import tensorflow.summary from tensorflow import random as tfrandom, int64 as tfint64 from ray import init as init_ray, shutdown as shutdown_ray from ray import tune from wandb.ray import WandbLogger from wandb import sweep as wandbsweep from wandb.apis import CommError as wandbCommError # less summaries are logged if MLENCRYPT_TB is TRUE (for efficiency) # TODO: use tf.summary.record_if? environ["MLENCRYPT_TB"] = str(tensorboard).upper() environ["MLENCRYPT_BARE"] = str(bare).upper() if getenv('MLENCRYPT_TB', 'FALSE') == 'TRUE' and \ getenv('MLENCRYPT_BARE', 'FALSE') == 'TRUE': raise ValueError('TensorBoard logging cannot be enabled in bare mode.') logdir = f'logs/hparams/{datetime.now()}' # "These results show that K = 3 is the optimal choice for the # cryptographic application of neural synchronization. K = 1 and K = 2 are # too insecure in regard to the geometric attack. And for K > 3 the effort # of A and B grows exponentially with increasing L, while the simple attack # is quite successful in the limit K -> infinity. Consequently, one should # only use Tree Parity Machines with three hidden units for the neural # key-exchange protocol." (Ruttor, 2006) # https://arxiv.org/pdf/0711.2411.pdf#page=59 update_rules = [ 'random-same', # 'random-different-A-B-E', 'random-different-A-B', 'hebbian', 'anti_hebbian', 'random_walk' ] K_bounds = {'min': 4, 'max': 8} N_bounds = {'min': 4, 'max': 8} L_bounds = {'min': 4, 'max': 8} # TODO: don't use *_bounds.values() since .values doesn't preserve order def get_session_num(logdir): current_runs = glob(join(logdir, "run-*")) if current_runs: last_run_path = current_runs[-1] last_run_session_num = int(last_run_path.split('-')[-1]) return last_run_session_num + 1 else: # there are no runs yet, start at 0 return 0 def trainable(config, reporter): """ Args: config (dict): Parameters provided from the search algorithm or variant generation. """ if not isinstance(config['update_rule'], str): update_rule = update_rules[int(config['update_rule'])] else: update_rule = config['update_rule'] K, N, L = int(config['K']), int(config['N']), int(config['L']) run_name = f"run-{get_session_num(logdir)}" run_logdir = join(logdir, run_name) # for each attack, the TPMs should start with the same weights initial_weights_tensors = get_initial_weights(K, N, L) training_steps_ls = {} eve_scores_ls = {} losses_ls = {} # for each attack, the TPMs should use the same inputs seed = tfrandom.uniform([], minval=0, maxval=tfint64.max, dtype=tfint64).numpy() for attack in ['none', 'geometric']: initial_weights = { tpm: weights_tensor_to_variable(weights, tpm) for tpm, weights in initial_weights_tensors.items() } tfrandom.set_seed(seed) if tensorboard: attack_logdir = join(run_logdir, attack) attack_writer = tensorflow.summary.create_file_writer( attack_logdir) with attack_writer.as_default(): training_steps, sync_scores, loss = run( update_rule, K, N, L, attack, initial_weights) else: training_steps, sync_scores, loss = run( update_rule, K, N, L, attack, initial_weights) training_steps_ls[attack] = training_steps eve_scores_ls[attack] = sync_scores losses_ls[attack] = loss avg_training_steps = tensorflow.math.reduce_mean( list(training_steps_ls.values())) avg_eve_score = tensorflow.math.reduce_mean( list(eve_scores_ls.values())) mean_loss = tensorflow.math.reduce_mean(list(losses_ls.values())) reporter( avg_training_steps=avg_training_steps.numpy(), avg_eve_score=avg_eve_score.numpy(), mean_loss=mean_loss.numpy(), done=True, ) if algorithm == 'hyperopt': from hyperopt import hp as hyperopt from hyperopt.pyll.base import scope from ray.tune.suggest.hyperopt import HyperOptSearch space = { 'update_rule': hyperopt.choice( 'update_rule', update_rules, ), 'K': scope.int(hyperopt.quniform('K', *K_bounds.values(), q=1)), 'N': scope.int(hyperopt.quniform('N', *N_bounds.values(), q=1)), 'L': scope.int(hyperopt.quniform('L', *L_bounds.values(), q=1)), } algo = HyperOptSearch( space, metric='mean_loss', mode='min', points_to_evaluate=[ { 'update_rule': 0, 'K': 3, 'N': 16, 'L': 8 }, { 'update_rule': 0, 'K': 8, 'N': 16, 'L': 8 }, { 'update_rule': 0, 'K': 8, 'N': 16, 'L': 128 }, ], ) elif algorithm == 'bayesopt': from ray.tune.suggest.bayesopt import BayesOptSearch space = { 'update_rule': (0, len(update_rules)), 'K': tuple(K_bounds.values()), 'N': tuple(N_bounds.values()), 'L': tuple(L_bounds.values()), } algo = BayesOptSearch( space, metric="mean_loss", mode="min", # TODO: what is utility_kwargs for and why is it needed? utility_kwargs={ "kind": "ucb", "kappa": 2.5, "xi": 0.0 }) elif algorithm == 'nevergrad': from ray.tune.suggest.nevergrad import NevergradSearch from nevergrad import optimizers from nevergrad import p as ngp algo = NevergradSearch( optimizers.TwoPointsDE( ngp.Instrumentation( update_rule=ngp.Choice(update_rules), K=ngp.Scalar(lower=K_bounds['min'], upper=K_bounds['max']).set_integer_casting(), N=ngp.Scalar(lower=N_bounds['min'], upper=N_bounds['max']).set_integer_casting(), L=ngp.Scalar(lower=L_bounds['min'], upper=L_bounds['max']).set_integer_casting(), )), None, # since the optimizer is already instrumented with kwargs metric="mean_loss", mode="min") elif algorithm == 'skopt': from skopt import Optimizer from ray.tune.suggest.skopt import SkOptSearch optimizer = Optimizer([ update_rules, tuple(K_bounds.values()), tuple(N_bounds.values()), tuple(L_bounds.values()) ]) algo = SkOptSearch( optimizer, ["update_rule", "K", "N", "L"], metric="mean_loss", mode="min", points_to_evaluate=[ ['random-same', 3, 16, 8], ['random-same', 8, 16, 8], ['random-same', 8, 16, 128], ], ) elif algorithm == 'dragonfly': # TODO: doesn't work from ray.tune.suggest.dragonfly import DragonflySearch from dragonfly.exd.experiment_caller import EuclideanFunctionCaller from dragonfly.opt.gp_bandit import EuclideanGPBandit # from dragonfly.exd.experiment_caller import CPFunctionCaller # from dragonfly.opt.gp_bandit import CPGPBandit from dragonfly import load_config domain_config = load_config({ "domain": [ { "name": "update_rule", "type": "discrete", "dim": 1, "items": update_rules }, { "name": "K", "type": "int", "min": K_bounds['min'], "max": K_bounds['max'], # "dim": 1 }, { "name": "N", "type": "int", "min": N_bounds['min'], "max": N_bounds['max'], # "dim": 1 }, { "name": "L", "type": "int", "min": L_bounds['min'], "max": L_bounds['max'], # "dim": 1 } ] }) func_caller = EuclideanFunctionCaller( None, domain_config.domain.list_of_domains[0]) optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True) algo = DragonflySearch( optimizer, metric="mean_loss", mode="min", points_to_evaluate=[ ['random-same', 3, 16, 8], ['random-same', 8, 16, 8], ['random-same', 8, 16, 128], ], ) elif algorithm == 'bohb': from ConfigSpace import ConfigurationSpace from ConfigSpace import hyperparameters as CSH from ray.tune.suggest.bohb import TuneBOHB config_space = ConfigurationSpace() config_space.add_hyperparameter( CSH.CategoricalHyperparameter("update_rule", choices=update_rules)) config_space.add_hyperparameter( CSH.UniformIntegerHyperparameter(name='K', lower=K_bounds['min'], upper=K_bounds['max'])) config_space.add_hyperparameter( CSH.UniformIntegerHyperparameter(name='N', lower=N_bounds['min'], upper=N_bounds['max'])) config_space.add_hyperparameter( CSH.UniformIntegerHyperparameter(name='L', lower=L_bounds['min'], upper=L_bounds['max'])) algo = TuneBOHB(config_space, metric="mean_loss", mode="min") elif algorithm == 'zoopt': from ray.tune.suggest.zoopt import ZOOptSearch from zoopt import ValueType space = { "update_rule": (ValueType.DISCRETE, range(0, len(update_rules)), False), "K": (ValueType.DISCRETE, range(K_bounds['min'], K_bounds['max'] + 1), True), "N": (ValueType.DISCRETE, range(N_bounds['min'], N_bounds['max'] + 1), True), "L": (ValueType.DISCRETE, range(L_bounds['min'], L_bounds['max'] + 1), True), } # TODO: change budget to a large value algo = ZOOptSearch(budget=10, dim_dict=space, metric="mean_loss", mode="min") # TODO: use more appropriate arguments for schedulers: # https://docs.ray.io/en/master/tune/api_docs/schedulers.html if scheduler == 'fifo': sched = None # Tune defaults to FIFO elif scheduler == 'pbt': from ray.tune.schedulers import PopulationBasedTraining from random import randint sched = PopulationBasedTraining( metric="mean_loss", mode="min", hyperparam_mutations={ "update_rule": update_rules, "K": lambda: randint(K_bounds['min'], K_bounds['max']), "N": lambda: randint(N_bounds['min'], N_bounds['max']), "L": lambda: randint(L_bounds['min'], L_bounds['max']), }) elif scheduler == 'ahb' or scheduler == 'asha': # https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#asha-tune-schedulers-ashascheduler from ray.tune.schedulers import AsyncHyperBandScheduler sched = AsyncHyperBandScheduler(metric="mean_loss", mode="min") elif scheduler == 'hb': from ray.tune.schedulers import HyperBandScheduler sched = HyperBandScheduler(metric="mean_loss", mode="min") elif algorithm == 'bohb' or scheduler == 'bohb': from ray.tune.schedulers import HyperBandForBOHB sched = HyperBandForBOHB(metric="mean_loss", mode="min") elif scheduler == 'msr': from ray.tune.schedulers import MedianStoppingRule sched = MedianStoppingRule(metric="mean_loss", mode="min") init_ray( address=getenv("ip_head"), redis_password=getenv('redis_password'), ) analysis = tune.run( trainable, name='mlencrypt_research', config={ "monitor": True, "env_config": { "wandb": { "project": "mlencrypt-research", "sync_tensorboard": True, }, }, }, # resources_per_trial={"cpu": 1, "gpu": 3}, local_dir='./ray_results', export_formats=['csv'], # TODO: add other formats? num_samples=num_samples, loggers=[ tune.logger.JsonLogger, tune.logger.CSVLogger, tune.logger.TBXLogger, WandbLogger ], search_alg=algo, scheduler=sched, queue_trials=True, ) try: wandbsweep(analysis) except wandbCommError: # see https://docs.wandb.com/sweeps/ray-tune#feature-compatibility pass best_config = analysis.get_best_config(metric='mean_loss', mode='min') print(f"Best config: {best_config}") shutdown_ray()
def create_optimization_algorithm(self, hyperparameter_space): """ Create optimization algorithm for Ray Tune. Currently, only tree parzen estimators, random search, gaussian processes and genetic algorithm based optimization using Ray Tune is supported. Parameters: space (dict, list or ray.tune.automl.search_space.SearchSpace): space of hyperparameters following the syntax required by the optimization algorithm. Returns: Optimization algorithm for Ray Tune. Raises: NotImplementedError: if self.optimization_type is other than ``tree_parzen_estimators``, ``random_search``, ``gaussian_processes`` or ``genetic_algorithm``. """ if self.optimization_type == 'tree_parzen_estimators': algorithm = HyperOptSearch(hyperparameter_space, max_concurrent=self.num_parallel_trials, metric='loss', mode=self.mode, n_initial_points=self.n_initial_points, random_state_seed=self.random_state, gamma=self.tpe_config.get( 'gamma', 0.25)) elif self.optimization_type == 'random_search': algorithm = HyperOptSearch(hyperparameter_space, max_concurrent=self.num_parallel_trials, metric='loss', mode=self.mode, random_state_seed=self.random_state) algorithm.algo = hpo.rand.suggest elif self.optimization_type == 'gaussian_processes': if not self.reload_trials: # the gaussian processes based optimization needs an # extra parameter self.gp_opt self.gp_opt = skopt.Optimizer( hyperparameter_space, n_initial_points=self.n_initial_points, base_estimator=self.gp_config.get('base_estimator', 'GP'), acq_func=self.gp_config.get('acq_function', 'gp_hedge'), acq_optimizer=self.gp_config.get('acq_optimizer', 'auto'), random_state=self.random_state, acq_func_kwargs={ 'xi': self.gp_config.get('xi', 0.01), 'kappa': self.gp_config.get('kappa', 1.96) }) hyperparams_names = [key for key in self.hyperparams_to_optimize] algorithm = SkOptSearch(self.gp_opt, hyperparams_names, max_concurrent=self.num_parallel_trials, metric='loss', mode=self.mode) elif self.optimization_type == 'genetic_algorithm': algorithm = GeneticSearch( hyperparameter_space, reward_attr='loss', max_generation=self.ga_config['max_generation'], population_size=self.ga_config['population_size'], population_decay=self.ga_config.get('population_decay', 0.95), keep_top_ratio=self.ga_config.get('keep_top_ratio', 0.2), selection_bound=self.ga_config.get('selection_bound', 0.4), crossover_bound=self.ga_config.get('crossover_bound', 0.4)) else: raise NotImplementedError( 'Other optimization types are not supported yet') return algorithm
def main_NN(data, num_samples=15, max_num_epochs=30, metric = 'loss', mode = 'min', checkpoint_dir = "C:\\Users\\nhian\\Desktop\\ray_results_12_13\\", tune_path = "C:/Users/nhian/Dropbox/UCLA MFE/Spring 2020/AFP/ray_results/", experiment_name = "experiment"+str(date.today()), trial_name = None): ''' data = tuple of (X, y) dataset used for training and validation num_samples = samples to search from search space max_num_epochs = max number of epochs to train the NN search over NN hyperspace defined by config max_num_epochs = max epochs for ASHAScheduler to terminate training num_samples = num trials trial_name= current trial name trial_dir = same as trial name ''' # config = { # "num_neurons": tune.choice([16, 32, 64, 128]), # "num_hidden": tune.choice([2,3,4]), # "activation_fn" : tune.choice([F.relu, F.leaky_relu]), # "batch_size": tune.choice([32]), # "lr": tune.loguniform(1e-4, 1e-1) # } bayes_searchspace = { "num_neurons": Categorical([int(x) for x in 2**np.arange(4,8)]), "num_hidden": Integer(2, 4, 'uniform'), "activation_fn" : Categorical([F.relu]), # "batch_size": Integer(16, 32),#Categorical([int(16), int(32)]), "lr": Real(1e-4, 1e-2, 'log-uniform') } # bayesopt = BayesOptSearch(metric="accuracy", mode="max") skopt_search = SkOptSearch(space = bayes_searchspace, metric="accuracy", mode="max") ''' can set metric/mode in scheduler or tune.run ASHA scheduler can set max_num_epochs grace_period = min number of iterations before stopping ''' scheduler = ASHAScheduler( metric= metric, #loss mode= mode, #min time_attr='training_iteration', max_t=max_num_epochs, grace_period=10, reduction_factor=2) ''' CLIReporter for python console what to print to console ''' reporter = CLIReporter( parameter_columns=["lr", "num_neurons", "num_hidden"], metric_columns=["loss", "accuracy", "training_iteration"]) #get dataset x, y = data input_size = x.shape[1] #need to register function (trainable) (or use partial and fill out other arguments) # tune.register_trainable("fc_nn", # lambda cfg : trainNN(config = cfg, x = x, y = y, # input_size = input_size, max_num_epochs = max_num_epochs)) def trial_name_string(trial): return trial_name+str(trial.trial_id) result = tune.run( partial(trainNN, x = x, y = y, input_size = input_size, max_num_epochs = max_num_epochs, checkpoint_dir = checkpoint_dir), resources_per_trial={"cpu": 3, "gpu": 0}, # config=config, search_alg = skopt_search, num_samples=num_samples, scheduler=scheduler, reuse_actors = True, progress_reporter=reporter, name= experiment_name, local_dir = tune_path, trial_name_creator = trial_name_string, trial_dirname_creator = trial_name_string) #get best trial best_trial = result.get_best_trial(metric, mode, "last") #"accuracy" , max print("Best trial config: {}".format(best_trial.config)) print("Best trial final validation loss: {}".format( best_trial.last_result["loss"])) print("Best trial final validation accuracy: {}".format( best_trial.last_result["accuracy"])) best_trained_model = net(input_size, best_trial.config["num_neurons"], best_trial.config["num_hidden"], best_trial.config["activation_fn"]) # best_checkpoint_dir = best_trial.checkpoint.value # model_state, optimizer_state = torch.load(os.path.join( # checkpoint_dir, "checkpoint")) # best_trained_model.load_state_dict(model_state) device = "cpu" if torch.cuda.is_available(): device = "cuda:0" # if gpus_per_trial > 1: # best_trained_model = nn.DataParallel(best_trained_model) best_trained_model.to(device) return best_trained_model, result
def testConvertSkOpt(self): from ray.tune.suggest.skopt import SkOptSearch from skopt.space import Real, Integer # Grid search not supported, should raise ValueError with self.assertRaises(ValueError): SkOptSearch.convert_search_space( {"grid": tune.grid_search([0, 1])}) config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = SkOptSearch.convert_search_space(config) skopt_config = { "a": [2, 3, 4], "b/x": Integer(0, 5), "b/z": Real(1e-4, 1e-2, prior="log-uniform") } searcher1 = SkOptSearch(space=converted_config, metric="a", mode="max") searcher2 = SkOptSearch(space=skopt_config, metric="a", mode="max") np.random.seed(1234) config1 = searcher1.suggest("0") np.random.seed(1234) config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = SkOptSearch(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] self.assertIn(trial.config["a"], [2, 3, 4]) self.assertEqual(trial.config["b"]["y"], 4) mixed_config = {"a": tune.uniform(5, 6), "b": (8, 9)} searcher = SkOptSearch(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)
help="Specify the out csv filename.", required=True) args = parser.parse_args() # Defining the hyperspace hyperparameters = [ (0.00001, 0.1), # learning_rate (0.2, 0.9), # dropout (10, 100), # epochs (10, 1000) ] # batch size space = create_hyperspace(hyperparameters) # Perform runs and aggregate results results = [] for section in tqdm(space): # create a skopt gp minimize object optimizer = Optimizer(section) search_algo = SkOptSearch( optimizer, ['learning_rate', 'dropout', 'epochs', 'batch_size'], metric='average_res', mode='max') analysis = tune.run(multi_train, search_alg=search_algo, num_samples=50, resources_per_trial={'gpu': 1}) results.append(analysis) all_pt_results = results[0].results_df for i in range(1, len(results)): all_pt_results = all_pt_results.append(results[i].results_df) all_pt_results.to_csv(args.out)
labels, epsilons=epsilons) robust_accuracy = 1 - success.cpu().numpy().astype(float).flatten().mean( axis=-1) # res test[0] reports the loss from the evaluation, res_test[1] reports the accuracy tune.report(robust_acc=robust_accuracy) return robust_accuracy if __name__ == "__main__": results = [] for section in tqdm(space): # create a skopt gp minimize object optimizer = Optimizer(section) search_algo = SkOptSearch( optimizer, ['learning_rate', 'dropout', 'epochs', 'batch_size'], metric='robust_acc', mode='max') # not using a gpu because running on local analysis = tune.run(mnist_pt_objective, search_alg=search_algo, num_samples=20, resources_per_trial={'gpu': 1}) results.append(analysis) print(type(results[0])) all_pt_results = results[0].results_df for i in range(1, len(results)): all_pt_results = all_pt_results.append(results[i].results_df) all_pt_results.to_csv('pt_fool_saltandpepper.csv')
{ "width": 10, "height": 0, "activation": "relu" # Activation will be relu }, { "width": 15, "height": -20, "activation": "tanh" # Activation will be tanh } ] known_rewards = [-189, -1144] algo = SkOptSearch( # parameter_names=space.keys(), # If you want to set the space # parameter_ranges=space.values(), # If you want to set the space points_to_evaluate=previously_run_params, evaluated_rewards=known_rewards) algo = ConcurrencyLimiter(algo, max_concurrent=4) scheduler = AsyncHyperBandScheduler() analysis = tune.run(easy_objective, metric="mean_loss", mode="min", name="skopt_exp_with_warmstart", search_alg=algo, scheduler=scheduler, num_samples=10 if args.smoke_test else 50, config={ "steps": 100,
def _tune_run(self, config, resources_per_trial): """Wrapper to call ``tune.run``. Multiple estimators are generated when early stopping is possible, whereas a single estimator is generated when early stopping is not possible. Args: config (dict): Configurations such as hyperparameters to run ``tune.run`` on. resources_per_trial (dict): Resources to use per trial within Ray. Accepted keys are `cpu`, `gpu` and custom resources, and values are integers specifying the number of each resource to use. Returns: analysis (`ExperimentAnalysis`): Object returned by `tune.run`. """ stop_condition = {"training_iteration": self.max_iters} if self.early_stopping is not None: config["estimator_list"] = [ clone(self.estimator) for _ in range(self.n_splits) ] if hasattr(self.early_stopping, "_max_t_attr"): # we want to delegate stopping to schedulers which # support it, but we want it to stop eventually, just in case # the solution is to make the stop condition very big stop_condition = {"training_iteration": self.max_iters * 10} else: config["estimator_list"] = [self.estimator] if self.search_optimization == "random": run_args = dict(scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop=stop_condition, num_samples=self.num_samples, config=config, fail_fast=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) if isinstance(self.param_distributions, list): run_args["search_alg"] = RandomListSearcher( self.param_distributions) analysis = tune.run(_Trainable, **run_args) return analysis elif self.search_optimization == "bayesian": from skopt import Optimizer from ray.tune.suggest.skopt import SkOptSearch hyperparameter_names, spaces = self._get_skopt_params() search_algo = SkOptSearch(Optimizer(spaces), hyperparameter_names, metric="average_test_score", **self.search_kwargs) elif self.search_optimization == "bohb": from ray.tune.suggest.bohb import TuneBOHB config_space = self._get_bohb_config_space() search_algo = TuneBOHB(config_space, metric="average_test_score", mode="max", **self.search_kwargs) elif self.search_optimization == "optuna": from ray.tune.suggest.optuna import OptunaSearch config_space = self._get_optuna_params() search_algo = OptunaSearch(config_space, metric="average_test_score", mode="max", **self.search_kwargs) elif self.search_optimization == "hyperopt": from ray.tune.suggest.hyperopt import HyperOptSearch config_space = self._get_hyperopt_params() search_algo = HyperOptSearch(config_space, metric="average_test_score", mode="max", **self.search_kwargs) if isinstance(self.n_jobs, int) and self.n_jobs > 0: search_algo = ConcurrencyLimiter(search_algo, max_concurrent=self.n_jobs) analysis = tune.run(_Trainable, search_alg=search_algo, scheduler=self.early_stopping, reuse_actors=True, verbose=self.verbose, stop=stop_condition, num_samples=self.num_samples, config=config, fail_fast=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) return analysis
def run(exp_ident, exp_name, metric, tune_mode, spec, tune_run_kwargs, use_skopt, skopt_search_mode, skopt_space, skopt_ref_configs): spec = sacred_copy(spec) log_dir = tune_ex.observers[0].dir ray.init() def trainable_function(config): # "config" is passed in by Ray Tune run_exp(config, log_dir, exp_ident) if use_skopt: assert skopt_search_mode in {'min', 'max'}, \ 'skopt_search_mode must be "min" or "max", as appropriate for ' \ 'the metric being optimised' assert len(skopt_space) > 0, "was passed an empty skopt_space" # do some sacred_copy() calls to ensure that we don't accidentally put # a ReadOnlyDict or ReadOnlyList into our optimizer skopt_space = sacred_copy(skopt_space) skopt_search_mode = sacred_copy(skopt_search_mode) skopt_ref_configs = sacred_copy(skopt_ref_configs) metric = sacred_copy(metric) sorted_space = collections.OrderedDict([ (key, value) for key, value in sorted(skopt_space.items()) ]) for k, v in list(sorted_space.items()): # Cast each value in sorted_space to a skopt Dimension object, then # make the name of the Dimension object match the corresponding # key. This is the step that converts tuple ranges---like `(1e-3, # 2.0, 'log-uniform')`---to actual skopt `Space` objects. try: new_v = skopt.space.check_dimension(v) except ValueError: # Raise actually-informative value error instead raise ValueError(f"Dimension issue: k:{k} v: {v}") new_v.name = k sorted_space[k] = new_v skopt_optimiser = skopt.optimizer.Optimizer([*sorted_space.values()], base_estimator='RF') algo = SkOptSearch(skopt_optimiser, list(sorted_space.keys()), metric=metric, mode=skopt_search_mode, points_to_evaluate=[[ ref_config_dict[k] for k in sorted_space.keys() ] for ref_config_dict in skopt_ref_configs]) tune_run_kwargs = { 'search_alg': algo, 'scheduler': CheckpointFIFOScheduler(algo), **tune_run_kwargs, } # completely remove 'spec' if spec: print("Will ignore everything in 'spec' argument") spec = {} tune_run = tune.run( trainable_function, name=exp_name, config=spec, local_dir=tune_ex.observers[0].dir, **tune_run_kwargs, ) best_config = tune_run.get_best_config(metric=metric, mode=tune_mode) print(f"Best config is: {best_config}") print("Results available at: ") print(tune_run._get_trial_paths())
def run_ray_tune(ray_address): sk_space = collections.OrderedDict() sk_space['disc_up_per_iter'] = (2, 12) # small values don't work sk_space['sampler_time_steps'] = (8, 20) # small is okay? sk_space['sampler_batch_envs'] = (8, 24) # bigger = better? sk_space['ppo_lr'] = (1e-6, 1e-3, 'log-uniform') sk_space['ppo_gamma'] = (0.9, 1.0, 'log-uniform') sk_space['ppo_lambda'] = (0.9, 1.0, 'log-uniform') sk_space['ppo_ent'] = (1e-6, 1e-4, 'log-uniform') sk_space['ppo_adv_clip'] = (0.05, 0.2, 'uniform') sk_space['add_preproc'] = ['LoRes4E', 'LoRes3EA'] # allow us to have smaller batches or run more of them sk_space['ppo_minibatches'] = [4, 6] sk_space['ppo_epochs'] = [2, 10] sk_space['ppo_use_bn'] = [True, False] sk_space['ppo_aug'] = ['none', 'all', 'crop'] # things I'm commenting out for simplicity: # sk_space['bc_loss'] = ['0.0', str(int(1e-3)), str(1)] # sk_space['ppo_use_bn'] = [True, False] # things that don't matter that much: # sk_space['omit_noop'] = [True, False] # ??? # sk_space['disc_lr'] = (1e-5, 5e-4, 'log-uniform') # fix to 1e-4 # sk_space['disc_use_act'] = [True, False] # fix to True # sk_space['disc_all_frames'] = [True, False] # fix to True # sk_space['disc_replay_mult'] = opt_space.Integer(1, 32, 'log-uniform') # fix to 4 # noqa: E501 # sk_space['ppo_norm_adv'] = [True, False] # fix to False known_working = { 'disc_up_per_iter': [4, 2], 'sampler_time_steps': [16, 16], 'sampler_batch_envs': [32, 12], # 'bc_loss': [0.0, 0.0], 'ppo_lr': [2.5e-4, 2e-4], 'ppo_adv_clip': [0.05, 0.1], 'ppo_minibatches': [4, 5], 'ppo_epochs': [4, 6], 'ppo_use_bn': [False, False], 'ppo_aug': ['none', 'none'], 'ppo_gamma': [0.95, 0.9], 'ppo_lambda': [0.95, 0.9], 'ppo_ent': [1e-5, 1.2e-5], 'add_preproc': ['LoRes4E', 'LoRes4E'] # things that I'm removing because they'll take too much time # 'omit_noop': [True], # things that don't matter much: # 'disc_lr': [1e-4], # 'disc_use_act': [True], # 'disc_all_frames': [True], # 'disc_replay_mult': [4], # 'ppo_norm_adv': [False], } for k, v in list(sk_space.items()): new_v = opt_space.check_dimension(v) new_v.name = k sk_space[k] = new_v sk_optimiser = Optimizer(list(sk_space.values()), base_estimator='GP') n_known_working, = set(map(len, known_working.values())) search_alg = SkOptSearch( sk_optimiser, sk_space.keys(), max_concurrent=8, # XXX figure out how to make this configurable metric='hp_score', mode='max', points_to_evaluate=[[known_working[k][i] for k in sk_space] for i in range(n_known_working)]) if ray_address: ray.init(redis_address=ray_address) tune.run( ray_tune_trial, search_alg=search_alg, local_dir='ray-tune-results', resources_per_trial={"gpu": 0.24}, # this could be 2 days to a week of runs, depending on the env num_samples=200, scheduler=CheckpointFIFOScheduler(search_alg))