def set_algorithm(experiment_name, config): ''' Configure search algorithm. ''' if args.algorithm == 'hyperopt': algorithm = HyperOptSearch(points_to_evaluate=best_params) elif args.algorithm == 'ax': ax_client = AxClient(enforce_sequential_optimization=False) ax_client.create_experiment(name=experiment_name, parameters=config, objective_name="minimum", minimize=True) algorithm = AxSearch(ax_client=ax_client, points_to_evaluate=best_params) elif args.algorithm == 'nevergrad': algorithm = NevergradSearch( points_to_evaluate=best_params, optimizer=ng.optimizers.registry["PortfolioDiscreteOnePlusOne"]) elif args.algorithm == 'optuna': algorithm = OptunaSearch(points_to_evaluate=best_params, seed=args.seed) elif args.algorithm == 'pbt': algorithm = PopulationBasedTraining( time_attr="training_iteration", perturbation_interval=args.perturbation, hyperparam_mutations=config, synch=True) elif args.algorithm == 'random': algorithm = BasicVariantGenerator(max_concurrent=args.jobs) if args.algorithm not in ['random', 'pbt']: algorithm = ConcurrencyLimiter(algorithm, max_concurrent=args.jobs) return algorithm
def testNevergrad(self): from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng searcher = NevergradSearch(space=self.config, metric=self.metric_name, mode="max", optimizer=ng.optimizers.RandomSearch) self._save(searcher) searcher = NevergradSearch(space=self.config, metric=self.metric_name, mode="max", optimizer=ng.optimizers.RandomSearch) self._restore(searcher)
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None): config = create_config(backtest_config) n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1) session_dirpath = make_get_filepath(os.path.join('reports', backtest_config['exchange'], backtest_config['symbol'], f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", '')) iters = 10 if 'iters' in backtest_config: iters = backtest_config['iters'] else: print('Parameter iters should be defined in the configuration. Defaulting to 10.') num_cpus = 2 if 'num_cpus' in backtest_config: num_cpus = backtest_config['num_cpus'] else: print('Parameter num_cpus should be defined in the configuration. Defaulting to 2.') n_particles = 10 if 'n_particles' in backtest_config: n_particles = backtest_config['n_particles'] phi1 = 1.4962 phi2 = 1.4962 omega = 0.7298 if 'options' in backtest_config: phi1 = backtest_config['options']['c1'] phi2 = backtest_config['options']['c2'] omega = backtest_config['options']['w'] current_best_params = [] if current_best: if type(current_best) == list: for c in current_best: c = clean_start_config(c, config, backtest_config['ranges']) current_best_params.append(c) else: current_best = clean_start_config(current_best, config, backtest_config['ranges']) current_best_params.append(current_best) ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False) pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2) algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params) algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus) scheduler = AsyncHyperBandScheduler() analysis = tune.run(tune.with_parameters(backtest, ticks=ticks), metric='objective', mode='max', name='search', search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1, reuse_actors=True, local_dir=session_dirpath, progress_reporter=LogReporter(metric_columns=['daily_gain', 'closest_liquidation', 'objective'], parameter_columns=[k for k in backtest_config['ranges']])) ray.shutdown() df = analysis.results_df df.reset_index(inplace=True) df.drop(columns=['trial_id', 'time_this_iter_s', 'done', 'timesteps_total', 'episodes_total', 'training_iteration', 'experiment_id', 'date', 'timestamp', 'time_total_s', 'pid', 'hostname', 'node_ip', 'time_since_restore', 'timesteps_since_restore', 'iterations_since_restore', 'experiment_tag'], inplace=True) df.to_csv(os.path.join(backtest_config['session_dirpath'], 'results.csv'), index=False) print('Best candidate found:') pprint.pprint(analysis.best_config) plot_wrap(backtest_config, ticks, clean_result_config(analysis.best_config)) return analysis
def testConvergenceNevergrad(self): from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng np.random.seed(0) searcher = NevergradSearch(optimizer=ng.optimizers.PSO) analysis = self._testConvergence(searcher, patience=50, top=5) assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-3)
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None): config = create_config(backtest_config) n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1) session_dirpath = make_get_filepath(os.path.join('reports', backtest_config['exchange'], backtest_config['symbol'], f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", '')) iters = 10 if 'iters' in backtest_config: iters = backtest_config['iters'] else: print('Parameter iters should be defined in the configuration. Defaulting to 10.') num_cpus = 2 if 'num_cpus' in backtest_config: num_cpus = backtest_config['num_cpus'] else: print('Parameter num_cpus should be defined in the configuration. Defaulting to 2.') n_particles = 10 if 'n_particles' in backtest_config: n_particles = backtest_config['n_particles'] phi1 = 1.4962 phi2 = 1.4962 omega = 0.7298 if 'options' in backtest_config: phi1 = backtest_config['options']['c1'] phi2 = backtest_config['options']['c2'] omega = backtest_config['options']['w'] current_best_params = [] if current_best: if type(current_best) == list: for c in current_best: c = clean_start_config(c, config, backtest_config['ranges']) current_best_params.append(c) else: current_best = clean_start_config(current_best, config, backtest_config['ranges']) current_best_params.append(current_best) ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False) pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2) algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params) algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus) scheduler = AsyncHyperBandScheduler() analysis = tune.run(tune.with_parameters(wrap_backtest, ticks=ticks), metric='objective', mode='max', name='search', search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1, reuse_actors=True, local_dir=session_dirpath, progress_reporter=LogReporter(metric_columns=['daily_gain', 'closest_liquidation', 'max_hours_between_fills', 'objective'], parameter_columns=[k for k in backtest_config['ranges'] if type( config[k]) == ray.tune.sample.Float or type( config[k]) == ray.tune.sample.Integer])) ray.shutdown() return analysis
def set_basic_conf(self): instrumentation = 2 parameter_names = ["height", "width"] optimizer = optimizerlib.OnePlusOne(instrumentation) def cost(space, reporter): reporter( mean_loss=(space["height"] - 14)**2 - abs(space["width"] - 3)) search_alg = NevergradSearch( optimizer, parameter_names, metric="mean_loss", mode="min") return search_alg, cost
def set_basic_conf(self): instrumentation = 2 parameter_names = ["height", "width"] optimizer = optimizerlib.OnePlusOne(instrumentation) def cost(space, reporter): reporter(loss=(space["height"] - 14)**2 - abs(space["width"] - 3)) search_alg = NevergradSearch( optimizer, parameter_names, metric="loss", mode="min", max_concurrent=1000, # Here to avoid breaking back-compat. ) return search_alg, cost
def testNevergrad(self): from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng np.random.seed(2020) # At least one nan, inf, -inf and float out = tune.run( _invalid_objective, search_alg=NevergradSearch(optimizer=ng.optimizers.RandomSearch), config=self.config, mode="max", num_samples=16, reuse_actors=False) best_trial = out.best_trial self.assertLessEqual(best_trial.config["report"], 2.0)
def testConvertNevergrad(self): from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = NevergradSearch.convert_search_space(config) nevergrad_config = ng.p.Dict( a=ng.p.Choice([2, 3, 4]), b=ng.p.Dict(x=ng.p.Scalar(lower=0, upper=5).set_integer_casting(), z=ng.p.Log(lower=1e-4, upper=1e-2))) searcher1 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne, space=converted_config) searcher2 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne, space=nevergrad_config) np.random.seed(1234) config1 = searcher1.suggest("0") np.random.seed(1234) config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = NevergradSearch(optimizer=ng.optimizers.OnePlusOne, metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4]
def testNevergradBestParams(self): from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng config = { "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), "b": tune.sample.Integer(0, 5), "c": tune.sample.Float(1e-4, 1e-1).loguniform() } best_params = [{ "metric": 1, "a": "t1", "b": 1, "c": 1e-1 }, { "metric": 2, "a": "t2", "b": 2, "c": 1e-2 }] searcher = NevergradSearch(optimizer=ng.optimizers.OnePlusOne, points_to_evaluate=best_params) analysis = tune.run(_mock_objective, config=config, metric="metric", mode="max", search_alg=searcher, num_samples=5) for i in range(len(best_params)): trial_config = analysis.trials[i].config trial_config_dict = { "metric": trial_config["metric"], "a": trial_config["a"], "b": trial_config["b"], "c": trial_config["c"] } self.assertDictEqual(trial_config_dict, best_params[i])
def cifar10_main(method='BlendSearch', num_samples=10, max_num_epochs=100, gpus_per_trial=2): data_dir = os.path.abspath("test/data") load_data(data_dir) # Download data for all trials before starting the run if method == 'BlendSearch': from flaml import tune else: from ray import tune if method in ['BlendSearch', 'BOHB', 'Optuna']: config = { "l1": tune.randint(2, 8), "l2": tune.randint(2, 8), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs, q=1), "batch_size": tune.randint(1, 4) #tune.choice([2, 4, 8, 16]) } else: config = { "l1": tune.randint(2, 9), "l2": tune.randint(2, 9), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1), "batch_size": tune.randint(1, 5) #tune.choice([2, 4, 8, 16]) } import ray time_budget_s = 3600 start_time = time.time() if method == 'BlendSearch': result = tune.run(ray.tune.with_parameters(train_cifar, data_dir=data_dir), init_config={ "l1": 2, "l2": 2, "num_epochs": 1, "batch_size": 4, }, metric="loss", mode="min", max_resource=max_num_epochs, min_resource=1, report_intermediate_result=True, resources_per_trial={ "cpu": 2, "gpu": gpus_per_trial }, config=config, local_dir='logs/', num_samples=num_samples, time_budget_s=time_budget_s, use_ray=True) else: if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB() scheduler = HyperBandForBOHB(max_t=max_num_epochs) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "l1": 2, "l2": 2, "num_epochs": 1, "batch_size": 4, }]) elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1) result = tune.run(tune.with_parameters(train_cifar, data_dir=data_dir), resources_per_trial={ "cpu": 2, "gpu": gpus_per_trial }, config=config, metric="loss", mode="min", num_samples=num_samples, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo) ray.shutdown() logger.info(f"method={method}") logger.info(f"n_samples={num_samples}") logger.info(f"time={time.time()-start_time}") best_trial = result.get_best_trial("loss", "min", "all") logger.info("Best trial config: {}".format(best_trial.config)) logger.info("Best trial final validation loss: {}".format( best_trial.metric_analysis["loss"]["min"])) logger.info("Best trial final validation accuracy: {}".format( best_trial.metric_analysis["accuracy"]["max"])) best_trained_model = Net(2**best_trial.config["l1"], 2**best_trial.config["l2"]) device = "cpu" if torch.cuda.is_available(): device = "cuda:0" if gpus_per_trial > 1: best_trained_model = nn.DataParallel(best_trained_model) best_trained_model.to(device) checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint") model_state, optimizer_state = torch.load(checkpoint_path) best_trained_model.load_state_dict(model_state) test_acc = _test_accuracy(best_trained_model, device) logger.info("Best trial test set accuracy: {}".format(test_acc))
def testConvertNevergrad(self): from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng # Grid search not supported, should raise ValueError with self.assertRaises(ValueError): NevergradSearch.convert_search_space( {"grid": tune.grid_search([0, 1])}) config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = NevergradSearch.convert_search_space(config) nevergrad_config = ng.p.Dict( a=ng.p.Choice([2, 3, 4]), b=ng.p.Dict(x=ng.p.Scalar(lower=0, upper=5).set_integer_casting(), z=ng.p.Log(lower=1e-4, upper=1e-2))) searcher1 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne, space=converted_config, metric="a", mode="max") searcher2 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne, space=nevergrad_config, metric="a", mode="max") np.random.seed(1234) config1 = searcher1.suggest("0") np.random.seed(1234) config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = NevergradSearch(optimizer=ng.optimizers.OnePlusOne, metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] assert trial.config["a"] in [2, 3, 4] mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix Nevergrad cfg and tune } searcher = NevergradSearch(space=mixed_config, optimizer=ng.optimizers.OnePlusOne, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)
def hparams(algorithm, scheduler, num_samples, tensorboard, bare): from glob import glob import tensorflow.summary from tensorflow import random as tfrandom, int64 as tfint64 from ray import init as init_ray, shutdown as shutdown_ray from ray import tune from wandb.ray import WandbLogger from wandb import sweep as wandbsweep from wandb.apis import CommError as wandbCommError # less summaries are logged if MLENCRYPT_TB is TRUE (for efficiency) # TODO: use tf.summary.record_if? environ["MLENCRYPT_TB"] = str(tensorboard).upper() environ["MLENCRYPT_BARE"] = str(bare).upper() if getenv('MLENCRYPT_TB', 'FALSE') == 'TRUE' and \ getenv('MLENCRYPT_BARE', 'FALSE') == 'TRUE': raise ValueError('TensorBoard logging cannot be enabled in bare mode.') logdir = f'logs/hparams/{datetime.now()}' # "These results show that K = 3 is the optimal choice for the # cryptographic application of neural synchronization. K = 1 and K = 2 are # too insecure in regard to the geometric attack. And for K > 3 the effort # of A and B grows exponentially with increasing L, while the simple attack # is quite successful in the limit K -> infinity. Consequently, one should # only use Tree Parity Machines with three hidden units for the neural # key-exchange protocol." (Ruttor, 2006) # https://arxiv.org/pdf/0711.2411.pdf#page=59 update_rules = [ 'random-same', # 'random-different-A-B-E', 'random-different-A-B', 'hebbian', 'anti_hebbian', 'random_walk' ] K_bounds = {'min': 4, 'max': 8} N_bounds = {'min': 4, 'max': 8} L_bounds = {'min': 4, 'max': 8} # TODO: don't use *_bounds.values() since .values doesn't preserve order def get_session_num(logdir): current_runs = glob(join(logdir, "run-*")) if current_runs: last_run_path = current_runs[-1] last_run_session_num = int(last_run_path.split('-')[-1]) return last_run_session_num + 1 else: # there are no runs yet, start at 0 return 0 def trainable(config, reporter): """ Args: config (dict): Parameters provided from the search algorithm or variant generation. """ if not isinstance(config['update_rule'], str): update_rule = update_rules[int(config['update_rule'])] else: update_rule = config['update_rule'] K, N, L = int(config['K']), int(config['N']), int(config['L']) run_name = f"run-{get_session_num(logdir)}" run_logdir = join(logdir, run_name) # for each attack, the TPMs should start with the same weights initial_weights_tensors = get_initial_weights(K, N, L) training_steps_ls = {} eve_scores_ls = {} losses_ls = {} # for each attack, the TPMs should use the same inputs seed = tfrandom.uniform([], minval=0, maxval=tfint64.max, dtype=tfint64).numpy() for attack in ['none', 'geometric']: initial_weights = { tpm: weights_tensor_to_variable(weights, tpm) for tpm, weights in initial_weights_tensors.items() } tfrandom.set_seed(seed) if tensorboard: attack_logdir = join(run_logdir, attack) attack_writer = tensorflow.summary.create_file_writer( attack_logdir) with attack_writer.as_default(): training_steps, sync_scores, loss = run( update_rule, K, N, L, attack, initial_weights) else: training_steps, sync_scores, loss = run( update_rule, K, N, L, attack, initial_weights) training_steps_ls[attack] = training_steps eve_scores_ls[attack] = sync_scores losses_ls[attack] = loss avg_training_steps = tensorflow.math.reduce_mean( list(training_steps_ls.values())) avg_eve_score = tensorflow.math.reduce_mean( list(eve_scores_ls.values())) mean_loss = tensorflow.math.reduce_mean(list(losses_ls.values())) reporter( avg_training_steps=avg_training_steps.numpy(), avg_eve_score=avg_eve_score.numpy(), mean_loss=mean_loss.numpy(), done=True, ) if algorithm == 'hyperopt': from hyperopt import hp as hyperopt from hyperopt.pyll.base import scope from ray.tune.suggest.hyperopt import HyperOptSearch space = { 'update_rule': hyperopt.choice( 'update_rule', update_rules, ), 'K': scope.int(hyperopt.quniform('K', *K_bounds.values(), q=1)), 'N': scope.int(hyperopt.quniform('N', *N_bounds.values(), q=1)), 'L': scope.int(hyperopt.quniform('L', *L_bounds.values(), q=1)), } algo = HyperOptSearch( space, metric='mean_loss', mode='min', points_to_evaluate=[ { 'update_rule': 0, 'K': 3, 'N': 16, 'L': 8 }, { 'update_rule': 0, 'K': 8, 'N': 16, 'L': 8 }, { 'update_rule': 0, 'K': 8, 'N': 16, 'L': 128 }, ], ) elif algorithm == 'bayesopt': from ray.tune.suggest.bayesopt import BayesOptSearch space = { 'update_rule': (0, len(update_rules)), 'K': tuple(K_bounds.values()), 'N': tuple(N_bounds.values()), 'L': tuple(L_bounds.values()), } algo = BayesOptSearch( space, metric="mean_loss", mode="min", # TODO: what is utility_kwargs for and why is it needed? utility_kwargs={ "kind": "ucb", "kappa": 2.5, "xi": 0.0 }) elif algorithm == 'nevergrad': from ray.tune.suggest.nevergrad import NevergradSearch from nevergrad import optimizers from nevergrad import p as ngp algo = NevergradSearch( optimizers.TwoPointsDE( ngp.Instrumentation( update_rule=ngp.Choice(update_rules), K=ngp.Scalar(lower=K_bounds['min'], upper=K_bounds['max']).set_integer_casting(), N=ngp.Scalar(lower=N_bounds['min'], upper=N_bounds['max']).set_integer_casting(), L=ngp.Scalar(lower=L_bounds['min'], upper=L_bounds['max']).set_integer_casting(), )), None, # since the optimizer is already instrumented with kwargs metric="mean_loss", mode="min") elif algorithm == 'skopt': from skopt import Optimizer from ray.tune.suggest.skopt import SkOptSearch optimizer = Optimizer([ update_rules, tuple(K_bounds.values()), tuple(N_bounds.values()), tuple(L_bounds.values()) ]) algo = SkOptSearch( optimizer, ["update_rule", "K", "N", "L"], metric="mean_loss", mode="min", points_to_evaluate=[ ['random-same', 3, 16, 8], ['random-same', 8, 16, 8], ['random-same', 8, 16, 128], ], ) elif algorithm == 'dragonfly': # TODO: doesn't work from ray.tune.suggest.dragonfly import DragonflySearch from dragonfly.exd.experiment_caller import EuclideanFunctionCaller from dragonfly.opt.gp_bandit import EuclideanGPBandit # from dragonfly.exd.experiment_caller import CPFunctionCaller # from dragonfly.opt.gp_bandit import CPGPBandit from dragonfly import load_config domain_config = load_config({ "domain": [ { "name": "update_rule", "type": "discrete", "dim": 1, "items": update_rules }, { "name": "K", "type": "int", "min": K_bounds['min'], "max": K_bounds['max'], # "dim": 1 }, { "name": "N", "type": "int", "min": N_bounds['min'], "max": N_bounds['max'], # "dim": 1 }, { "name": "L", "type": "int", "min": L_bounds['min'], "max": L_bounds['max'], # "dim": 1 } ] }) func_caller = EuclideanFunctionCaller( None, domain_config.domain.list_of_domains[0]) optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True) algo = DragonflySearch( optimizer, metric="mean_loss", mode="min", points_to_evaluate=[ ['random-same', 3, 16, 8], ['random-same', 8, 16, 8], ['random-same', 8, 16, 128], ], ) elif algorithm == 'bohb': from ConfigSpace import ConfigurationSpace from ConfigSpace import hyperparameters as CSH from ray.tune.suggest.bohb import TuneBOHB config_space = ConfigurationSpace() config_space.add_hyperparameter( CSH.CategoricalHyperparameter("update_rule", choices=update_rules)) config_space.add_hyperparameter( CSH.UniformIntegerHyperparameter(name='K', lower=K_bounds['min'], upper=K_bounds['max'])) config_space.add_hyperparameter( CSH.UniformIntegerHyperparameter(name='N', lower=N_bounds['min'], upper=N_bounds['max'])) config_space.add_hyperparameter( CSH.UniformIntegerHyperparameter(name='L', lower=L_bounds['min'], upper=L_bounds['max'])) algo = TuneBOHB(config_space, metric="mean_loss", mode="min") elif algorithm == 'zoopt': from ray.tune.suggest.zoopt import ZOOptSearch from zoopt import ValueType space = { "update_rule": (ValueType.DISCRETE, range(0, len(update_rules)), False), "K": (ValueType.DISCRETE, range(K_bounds['min'], K_bounds['max'] + 1), True), "N": (ValueType.DISCRETE, range(N_bounds['min'], N_bounds['max'] + 1), True), "L": (ValueType.DISCRETE, range(L_bounds['min'], L_bounds['max'] + 1), True), } # TODO: change budget to a large value algo = ZOOptSearch(budget=10, dim_dict=space, metric="mean_loss", mode="min") # TODO: use more appropriate arguments for schedulers: # https://docs.ray.io/en/master/tune/api_docs/schedulers.html if scheduler == 'fifo': sched = None # Tune defaults to FIFO elif scheduler == 'pbt': from ray.tune.schedulers import PopulationBasedTraining from random import randint sched = PopulationBasedTraining( metric="mean_loss", mode="min", hyperparam_mutations={ "update_rule": update_rules, "K": lambda: randint(K_bounds['min'], K_bounds['max']), "N": lambda: randint(N_bounds['min'], N_bounds['max']), "L": lambda: randint(L_bounds['min'], L_bounds['max']), }) elif scheduler == 'ahb' or scheduler == 'asha': # https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#asha-tune-schedulers-ashascheduler from ray.tune.schedulers import AsyncHyperBandScheduler sched = AsyncHyperBandScheduler(metric="mean_loss", mode="min") elif scheduler == 'hb': from ray.tune.schedulers import HyperBandScheduler sched = HyperBandScheduler(metric="mean_loss", mode="min") elif algorithm == 'bohb' or scheduler == 'bohb': from ray.tune.schedulers import HyperBandForBOHB sched = HyperBandForBOHB(metric="mean_loss", mode="min") elif scheduler == 'msr': from ray.tune.schedulers import MedianStoppingRule sched = MedianStoppingRule(metric="mean_loss", mode="min") init_ray( address=getenv("ip_head"), redis_password=getenv('redis_password'), ) analysis = tune.run( trainable, name='mlencrypt_research', config={ "monitor": True, "env_config": { "wandb": { "project": "mlencrypt-research", "sync_tensorboard": True, }, }, }, # resources_per_trial={"cpu": 1, "gpu": 3}, local_dir='./ray_results', export_formats=['csv'], # TODO: add other formats? num_samples=num_samples, loggers=[ tune.logger.JsonLogger, tune.logger.CSVLogger, tune.logger.TBXLogger, WandbLogger ], search_alg=algo, scheduler=sched, queue_trials=True, ) try: wandbsweep(analysis) except wandbCommError: # see https://docs.wandb.com/sweeps/ray-tune#feature-compatibility pass best_config = analysis.get_best_config(metric='mean_loss', mode='min') print(f"Best config: {best_config}") shutdown_ray()
def _test_xgboost(method="BlendSearch"): try: import ray except ImportError: return if method == "BlendSearch": from flaml import tune else: from ray import tune search_space = { "max_depth": tune.randint(1, 9) if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1), } max_iter = 10 for num_samples in [128]: time_budget_s = 60 for n_cpu in [2]: start_time = time.time() # ray.init(address='auto') if method == "BlendSearch": analysis = tune.run( train_breast_cancer, config=search_space, low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, scheduler="asha", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, use_ray=True, ) else: if "ASHA" == method: algo = None elif "BOHB" == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif "Optuna" == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif "CFO" == method: from flaml import CFO algo = CFO( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "CFOCat" == method: from flaml.searcher.cfo_cat import CFOCat algo = CFOCat( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "Dragonfly" == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif "SkOpt" == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif "Nevergrad" == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif "ZOOpt" == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples * n_cpu) elif "Ax" == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif "HyperOpt" == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != "BOHB": from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo, ) # # Load the best model checkpoint # import os # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss", "min", "all") accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def get_raytune_search_alg(raytune_cfg, seeds=False): if (raytune_cfg["sched"] == "pbt") or (raytune_cfg["sched"] == "pb2"): if raytune_cfg["search_alg"] is not None: print( "INFO: Using schedule '{}' is not compatible with Ray Tune search algorithms." .format(raytune_cfg["sched"])) print( "INFO: Uing the Ray Tune {} scheduler without search algorithm" .format(raytune_cfg["sched"])) return None if (raytune_cfg["sched"] == "bohb") or (raytune_cfg["sched"] == "BOHB"): print( "INFO: Using TuneBOHB search algorithm since it is required for BOHB shedule" ) if seeds: seed = 1234 else: seed = None return TuneBOHB(metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], seed=seed) # requires pip install bayesian-optimization if raytune_cfg["search_alg"] == "bayes": print("INFO: Using BayesOptSearch") return BayesOptSearch( metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], random_search_steps=raytune_cfg["bayes"]["n_random_steps"], ) # requires pip install hyperopt if raytune_cfg["search_alg"] == "hyperopt": print("INFO: Using HyperOptSearch") return HyperOptSearch( metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], n_initial_points=raytune_cfg["hyperopt"]["n_random_steps"], # points_to_evaluate=, ) if raytune_cfg["search_alg"] == "scikit": print("INFO: Using bayesian optimization from scikit-learn") return SkOptSearch( metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], convert_to_python=True, ) if raytune_cfg["search_alg"] == "nevergrad": print("INFO: Using bayesian optimization from nevergrad") return NevergradSearch( optimizer=ng.optimizers.BayesOptim( pca=False, init_budget=raytune_cfg["nevergrad"]["n_random_steps"]), metric=raytune_cfg["default_metric"], mode=raytune_cfg["default_mode"], ) # HEBO is not yet supported # if (raytune_cfg["search_alg"] == "hebo") or (raytune_cfg["search_alg"] == "HEBO"): # print("Using HEBOSearch") # return HEBOSearch( # metric=raytune_cfg["default_metric"], # mode=raytune_cfg["default_mode"], # # max_concurrent=8, # ) else: print("INFO: Not using any Ray Tune search algorithm") return None
params = {} params_bool = {} SELECT_LAMBDA_RANGE = [0.02, 0.03, 0.04, 0.05] SELECT_PRIOR_RANGE = [ 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15 ] SELECT_CONT_RANGE = [0.04, 0.06, 0.08, 0.10] if not os.path.exists('out'): os.makedirs('out') name = "randomMTM1" algo = NevergradSearch(optimizer=ng.optimizers.RandomSearch, metric="score", mode="max") algo = ConcurrencyLimiter(algo, max_concurrent=1) scheduler = ASHAScheduler() from ray import tune analysis = tune.run( train_MTM, name=name, scheduler=scheduler, reuse_actors=False, search_alg=algo, verbose=2, checkpoint_at_end=False, num_samples=32, # 64 # export_formats=[ExportFormat.MODEL],
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None): config = create_config(backtest_config) n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1) backtest_config['optimize_dirpath'] = os.path.join( backtest_config['optimize_dirpath'], ts_to_date(time())[:19].replace(':', ''), '') if 'iters' in backtest_config: iters = backtest_config['iters'] else: print( 'Parameter iters should be defined in the configuration. Defaulting to 10.' ) iters = 10 if 'num_cpus' in backtest_config: num_cpus = backtest_config['num_cpus'] else: print( 'Parameter num_cpus should be defined in the configuration. Defaulting to 2.' ) num_cpus = 2 n_particles = backtest_config[ 'n_particles'] if 'n_particles' in backtest_config else 10 phi1 = 1.4962 phi2 = 1.4962 omega = 0.7298 if 'options' in backtest_config: phi1 = backtest_config['options']['c1'] phi2 = backtest_config['options']['c2'] omega = backtest_config['options']['w'] current_best_params = [] if current_best: if type(current_best) == list: for c in current_best: c = clean_start_config(c, config, backtest_config['ranges']) if c not in current_best_params: current_best_params.append(c) else: current_best = clean_start_config(current_best, config, backtest_config['ranges']) current_best_params.append(current_best) ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False) pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2) algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params) algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus) scheduler = AsyncHyperBandScheduler() if 'wfo' in config and config['wfo']: print('\n\nwalk forward optimization\n\n') wfo = WFO(ticks, backtest_config, P_train=0.5).set_train_N(4) backtest_wrap = lambda config: tune_report(wfo.backtest(config)) else: print('\n\nsimple sliding window optimization\n\n') backtest_wrap = tune.with_parameters(simple_sliding_window_wrap, ticks=ticks) analysis = tune.run(backtest_wrap, metric='objective', mode='max', name='search', search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1, reuse_actors=True, local_dir=backtest_config['optimize_dirpath'], progress_reporter=LogReporter( metric_columns=[ 'daily_gain', 'closest_liquidation', 'max_hrs_no_fills', 'max_hrs_no_fills_same_side', 'objective' ], parameter_columns=[ k for k in backtest_config['ranges'] if type(config[k]) == ray.tune.sample.Float or type(config[k]) == ray.tune.sample.Integer ]), raise_on_failed_trial=False) ray.shutdown() return analysis
def cifar10_main(method="BlendSearch", num_samples=10, max_num_epochs=100, gpus_per_trial=1): data_dir = os.path.abspath("test/data") load_data(data_dir) # Download data for all trials before starting the run if method == "BlendSearch": from flaml import tune else: from ray import tune if method in ["BOHB"]: config = { "l1": tune.randint(2, 8), "l2": tune.randint(2, 8), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs, q=1), "batch_size": tune.randint(1, 4), } else: config = { "l1": tune.randint(2, 9), "l2": tune.randint(2, 9), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.loguniform(1, max_num_epochs), "batch_size": tune.randint(1, 5), } import ray time_budget_s = 600 np.random.seed(7654321) start_time = time.time() if method == "BlendSearch": result = tune.run( ray.tune.with_parameters(train_cifar, data_dir=data_dir), config=config, metric="loss", mode="min", low_cost_partial_config={"num_epochs": 1}, max_resource=max_num_epochs, min_resource=1, scheduler="asha", resources_per_trial={ "cpu": 1, "gpu": gpus_per_trial }, local_dir="logs/", num_samples=num_samples, time_budget_s=time_budget_s, use_ray=True, ) else: if "ASHA" == method: algo = None elif "BOHB" == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB() scheduler = HyperBandForBOHB(max_t=max_num_epochs) elif "Optuna" == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch(seed=10) elif "CFO" == method: from flaml import CFO algo = CFO(low_cost_partial_config={ "num_epochs": 1, }) elif "Nevergrad" == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) if method != "BOHB": from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1) result = tune.run( tune.with_parameters(train_cifar, data_dir=data_dir), resources_per_trial={ "cpu": 1, "gpu": gpus_per_trial }, config=config, metric="loss", mode="min", num_samples=num_samples, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo, ) ray.shutdown() logger.info(f"method={method}") logger.info(f"#trials={len(result.trials)}") logger.info(f"time={time.time()-start_time}") best_trial = result.get_best_trial("loss", "min", "all") logger.info("Best trial config: {}".format(best_trial.config)) logger.info("Best trial final validation loss: {}".format( best_trial.metric_analysis["loss"]["min"])) logger.info("Best trial final validation accuracy: {}".format( best_trial.metric_analysis["accuracy"]["max"])) best_trained_model = Net(2**best_trial.config["l1"], 2**best_trial.config["l2"]) device = "cpu" if torch.cuda.is_available(): device = "cuda:0" if gpus_per_trial > 1: best_trained_model = nn.DataParallel(best_trained_model) best_trained_model.to(device) checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint") model_state, optimizer_state = torch.load(checkpoint_path) best_trained_model.load_state_dict(model_state) test_acc = _test_accuracy(best_trained_model, device) logger.info("Best trial test set accuracy: {}".format(test_acc))
tune_kwargs = { "num_samples": 10 if args.smoke_test else 50, "config": { "steps": 100, "width": tune.uniform(0, 20), "height": tune.uniform(-100, 100), "activation": tune.choice(["relu", "tanh"]) } } # Optional: Pass the parameter space yourself # space = ng.p.Dict( # width=ng.p.Scalar(lower=0, upper=20), # height=ng.p.Scalar(lower=-100, upper=100), # activation=ng.p.Choice(choices=["relu", "tanh"]) # ) algo = NevergradSearch( optimizer=ng.optimizers.OnePlusOne, # space=space, # If you want to set the space manually metric="mean_loss", mode="min") scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min") tune.run(easy_objective, name="nevergrad", search_alg=algo, scheduler=scheduler, **tune_kwargs)
ray.init() config = { "num_samples": 10 if args.smoke_test else 50, "config": { "iterations": 100, }, "stop": { "timesteps_total": 100 } } instrumentation = 2 parameter_names = ["height", "width"] # With nevergrad v0.2.0+ the following is also possible: # from nevergrad import instrumentation as inst # instrumentation = inst.Instrumentation( # height=inst.var.Array(1).bounded(0, 200).asfloat(), # width=inst.var.OrderedDiscrete([0, 10, 20, 30, 40, 50])) # parameter_names = None # names are provided by the instrumentation optimizer = optimizerlib.OnePlusOne(instrumentation) algo = NevergradSearch(optimizer, parameter_names, max_concurrent=4, reward_attr="neg_mean_loss") scheduler = AsyncHyperBandScheduler(reward_attr="neg_mean_loss") run(easy_objective, name="nevergrad", search_alg=algo, scheduler=scheduler, **config)
config = { "num_samples": 10 if args.smoke_test else 50, "config": { "iterations": 100, }, "stop": { "timesteps_total": 100 } } instrumentation = 2 parameter_names = ["height", "width"] # With nevergrad v0.2.0+ the following is also possible: # from nevergrad import instrumentation as inst # instrumentation = inst.Instrumentation( # height=inst.var.Array(1).bounded(0, 200).asfloat(), # width=inst.var.OrderedDiscrete([0, 10, 20, 30, 40, 50])) # parameter_names = None # names are provided by the instrumentation optimizer = optimizerlib.OnePlusOne(instrumentation) algo = NevergradSearch(optimizer, parameter_names, max_concurrent=4, metric="mean_loss", mode="min") scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min") run(easy_objective, name="nevergrad", search_alg=algo, scheduler=scheduler, **config)
if __name__ == "__main__": import argparse from nevergrad.optimization import optimizerlib parser = argparse.ArgumentParser() parser.add_argument("--smoke-test", action="store_true", help="Finish quickly for testing") args, _ = parser.parse_known_args() ray.init() config = { "num_samples": 10 if args.smoke_test else 50, "config": { "iterations": 100, }, "stop": { "timesteps_total": 100 } } optimizer = optimizerlib.OnePlusOne(dimension=2) algo = NevergradSearch(optimizer, ["height", "width"], max_concurrent=4, reward_attr="neg_mean_loss") scheduler = AsyncHyperBandScheduler(reward_attr="neg_mean_loss") run(easy_objective, name="nevergrad", search_alg=algo, scheduler=scheduler, **config)
def backtest_tune(data: np.ndarray, config: dict, current_best: Union[dict, list] = None): memory = int(sys.getsizeof(data) * 1.2) virtual_memory = psutil.virtual_memory() print(f'data size in mb {memory / (1000 * 1000):.4f}') if (virtual_memory.available - memory) / virtual_memory.total < 0.1: print( "Available memory would drop below 10%. Please reduce the time span." ) return None config = create_config(config) print('tuning:') for k, v in config.items(): if type(v) in [ray.tune.sample.Float, ray.tune.sample.Integer]: print(k, (v.lower, v.upper)) phi1 = 1.4962 phi2 = 1.4962 omega = 0.7298 if 'options' in config: phi1 = config['options']['c1'] phi2 = config['options']['c2'] omega = config['options']['w'] current_best_params = [] if current_best is not None: if type(current_best) == list: for c in current_best: c = clean_start_config(c, config) if c not in current_best_params: current_best_params.append(c) else: current_best = clean_start_config(current_best, config) current_best_params.append(current_best) ray.init(num_cpus=config['num_cpus'], object_store_memory=memory if memory > 4000000000 else None) # , logging_level=logging.FATAL, log_to_driver=False) pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=config['n_particles'], omega=omega, phip=phi1, phig=phi2) algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params) algo = ConcurrencyLimiter(algo, max_concurrent=config['num_cpus']) scheduler = AsyncHyperBandScheduler() print('\n\nsimple sliding window optimization\n\n') parameter_columns = [] for side in ['long', 'shrt']: if config[f'{side}£enabled']: parameter_columns.append(f'{side}£grid_span') parameter_columns.append(f'{side}£eprice_pprice_diff') parameter_columns.append(f'{side}£eprice_exp_base') parameter_columns.append(f'{side}£secondary_pprice_diff') parameter_columns.append(f'{side}£min_markup') backtest_wrap = tune.with_parameters( simple_sliding_window_wrap, data=data, do_print=(config['print_slice_progress'] if 'print_slice_progress' in config else False)) analysis = tune.run( backtest_wrap, metric='obj', mode='max', name='search', search_alg=algo, scheduler=scheduler, num_samples=config['iters'], config=config, verbose=1, reuse_actors=True, local_dir=config['optimize_dirpath'], progress_reporter=LogReporter(metric_columns=[ 'min_adg', 'avg_adg', 'min_bkr', 'eqbal_ratio_min', 'hrs_stuck_max_l', 'hrs_stuck_max_s', 'pac_mean_l', 'pac_mean_s', 'n_slc', 'obj' ], parameter_columns=parameter_columns, max_report_frequency=30), raise_on_failed_trial=False) ray.shutdown() print('\nCleaning up temporary optimizer data...\n') try: shutil.rmtree(os.path.join(config['optimize_dirpath'], 'search')) except Exception as e: print('Failed cleaning up.') print(e) return analysis
from ray.tune.suggest.nevergrad import NevergradSearch budget = 400 opt_args = { 'scheduler.target_lrs': ng.var.Log(1e-4, 1e-2), 'scheduler.final_lr': ng.var.Log(1e-7, 1e-4), 'optimizer.weight_decay': ng.var.Log(1e-7, 1e-3), 'scheduler.exp_factor': ng.var.Scalar().bounded(-.5, 2), } inst = ng.Instrumentation(**opt_args) opt = ng.optimizers.registry['CMA'](instrumentation=inst, budget=budget) search_alg = NevergradSearch(opt, None, max_concurrent=4, reward_attr='valid_accuracy', mode='max') exp_args = { 'resume': False, 'search_alg': search_alg, 'scheduler': schedulers.FIFOScheduler(), 'stop': { 'is_finished': True }, 'verbose': True, 'num_samples': budget, 'checkpoint_at_end': False, 'resources_per_trial': { 'cpu': 2,
def _test_xgboost(method='BlendSearch'): try: import ray except ImportError: return if method == 'BlendSearch': from flaml import tune else: from ray import tune search_space = { # You can mix constants with search space objects. "max_depth": tune.randint(1, 8) if method in [ "BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1) } max_iter = 10 for num_samples in [256]: time_budget_s = 60 #None for n_cpu in [8]: start_time = time.time() ray.init(num_cpus=n_cpu, num_gpus=0) if method == 'BlendSearch': analysis = tune.run( train_breast_cancer, init_config={ "max_depth": 1, "min_child_weight": 3, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, report_intermediate_result=True, # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, use_ray=True) else: if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "max_depth": 1, "min_child_weight": 3, }], cat_hp_cost={ "min_child_weight": [6, 3, 2], }) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples*n_cpu) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler( max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo) ray.shutdown() # # Load the best model checkpoint # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss","min","all") accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def _test_distillbert(method='BlendSearch'): max_num_epoch = 64 num_samples = -1 time_budget_s = 10800 search_space = { # You can mix constants with search space objects. "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch), "learning_rate": flaml.tune.loguniform(1e-6, 1e-4), "adam_beta1": flaml.tune.uniform(0.8, 0.99), "adam_beta2": flaml.tune.loguniform(98e-2, 9999e-4), "adam_epsilon": flaml.tune.loguniform(1e-9, 1e-7), } start_time = time.time() ray.init(num_cpus=4, num_gpus=4) if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import tuneBOHB algo = tuneBOHB(max_concurrent=4) scheduler = HyperBandForBOHB(max_t=max_num_epoch) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "num_train_epochs": 1, }]) elif 'BlendSearch' == method: from flaml import BlendSearch algo = BlendSearch(points_to_evaluate=[{ "num_train_epochs": 1, }]) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1) scheduler = None analysis = ray.tune.run( train_distilbert, metric=HP_METRIC, mode=MODE, # You can add "gpu": 1 to allocate GPUs resources_per_trial={"gpu": 1}, config=search_space, local_dir='test/logs/', num_samples=num_samples, time_budget_s=time_budget_s, keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC, scheduler=scheduler, search_alg=algo) ray.shutdown() best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all") metric = best_trial.metric_analysis[HP_METRIC][MODE] logger.info(f"method={method}") logger.info(f"n_trials={len(analysis.trials)}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def _test_roberta(method='BlendSearch'): max_num_epoch = 100 num_samples = -1 time_budget_s = 3600 search_space = { # You can mix constants with search space objects. "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch), "learning_rate": flaml.tune.loguniform(1e-5, 3e-5), "weight_decay": flaml.tune.uniform(0, 0.3), "per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]), "seed": flaml.tune.choice([12, 22, 33, 42]), } start_time = time.time() ray.init(num_cpus=4, num_gpus=4) if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import tuneBOHB algo = tuneBOHB(max_concurrent=4) scheduler = HyperBandForBOHB(max_t=max_num_epoch) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "num_train_epochs": 1, "per_device_train_batch_size": 128, }]) elif 'BlendSearch' == method: from flaml import BlendSearch algo = BlendSearch( points_to_evaluate=[{ "num_train_epochs": 1, "per_device_train_batch_size": 128, }]) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch(max_concurrent=3) elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1) scheduler = None analysis = ray.tune.run(train_roberta, metric=HP_METRIC, mode=MODE, resources_per_trial={ "gpu": 4, "cpu": 4 }, config=search_space, local_dir='logs/', num_samples=num_samples, time_budget_s=time_budget_s, keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC, scheduler=scheduler, search_alg=algo) ray.shutdown() best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all") metric = best_trial.metric_analysis[HP_METRIC][MODE] logger.info(f"method={method}") logger.info(f"n_trials={len(analysis.trials)}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
args, _ = parser.parse_known_args() if args.server_address: import ray ray.init(f"ray://{args.server_address}") # Optional: Pass the parameter space yourself # space = ng.p.Dict( # width=ng.p.Scalar(lower=0, upper=20), # height=ng.p.Scalar(lower=-100, upper=100), # activation=ng.p.Choice(choices=["relu", "tanh"]) # ) algo = NevergradSearch( optimizer=ng.optimizers.OnePlusOne, # space=space, # If you want to set the space manually ) algo = ConcurrencyLimiter(algo, max_concurrent=4) scheduler = AsyncHyperBandScheduler() analysis = tune.run( easy_objective, metric="mean_loss", mode="min", name="nevergrad", search_alg=algo, scheduler=scheduler, num_samples=10 if args.smoke_test else 50, config={ "steps": 100,