def update_search_space(self, search_space): ''' Tuners are advised to support updating search space at run-time. If a tuner can only set search space once before generating first hyper-parameters, it should explicitly document this behaviour. search_space: JSON object created by experiment owner ''' config = {} for key, value in search_space: v = value.get("_value") _type = value['_type'] if _type == 'choice': config[key] = choice(v) elif _type == 'randint': config[key] = randint(v[0], v[1] - 1) elif _type == 'uniform': config[key] = uniform(v[0], v[1]) elif _type == 'quniform': config[key] = quniform(v[0], v[1], v[2]) elif _type == 'loguniform': config[key] = loguniform(v[0], v[1]) elif _type == 'qloguniform': config[key] = qloguniform(v[0], v[1], v[2]) elif _type == 'normal': config[key] = randn(v[1], v[2]) elif _type == 'qnormal': config[key] = qrandn(v[1], v[2], v[3]) else: raise ValueError( f'unsupported type in search_space {_type}') self._ls.set_search_properties(None, None, config) if self._gs is not None: self._gs.set_search_properties(None, None, config) self._init_search()
def update_search_space(self, search_space): """Required by NNI. Tuners are advised to support updating search space at run-time. If a tuner can only set search space once before generating first hyper-parameters, it should explicitly document this behaviour. Args: search_space: JSON object created by experiment owner. """ config = {} for key, value in search_space.items(): v = value.get("_value") _type = value["_type"] if _type == "choice": config[key] = choice(v) elif _type == "randint": config[key] = randint(*v) elif _type == "uniform": config[key] = uniform(*v) elif _type == "quniform": config[key] = quniform(*v) elif _type == "loguniform": config[key] = loguniform(*v) elif _type == "qloguniform": config[key] = qloguniform(*v) elif _type == "normal": config[key] = randn(*v) elif _type == "qnormal": config[key] = qrandn(*v) else: raise ValueError(f"unsupported type in search_space {_type}") # low_cost_partial_config is passed to constructor, # which is before update_search_space() is called init_config = self._ls.init_config add_cost_to_space(config, init_config, self._cat_hp_cost) self._ls = self.LocalSearch( init_config, self._ls.metric, self._mode, config, self._ls.resource_attr, self._ls.min_resource, self._ls.max_resource, self._ls.resource_multiple_factor, cost_attr=self.cost_attr, seed=self._ls.seed, ) if self._gs is not None: self._gs = GlobalSearch( space=config, metric=self._metric, mode=self._mode, sampler=self._gs._sampler, ) self._gs.space = config self._init_search()
def testTuneSampleAPI(self): config = { "func": tune.sample_from(lambda spec: spec.config.uniform * 0.01), "uniform": tune.uniform(-5, -1), "quniform": tune.quniform(3.2, 5.4, 0.2), "loguniform": tune.loguniform(1e-4, 1e-2), "qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5), "choice": tune.choice([2, 3, 4]), "randint": tune.randint(-9, 15), "qrandint": tune.qrandint(-21, 12, 3), "randn": tune.randn(10, 2), "qrandn": tune.qrandn(10, 2, 0.2), } for _, (_, generated) in zip( range(1000), generate_variants({ "config": config })): out = generated["config"] self.assertAlmostEqual(out["func"], out["uniform"] * 0.01) self.assertGreaterEqual(out["uniform"], -5) self.assertLess(out["uniform"], -1) self.assertGreaterEqual(out["quniform"], 3.2) self.assertLessEqual(out["quniform"], 5.4) self.assertAlmostEqual(out["quniform"] / 0.2, round(out["quniform"] / 0.2)) self.assertGreaterEqual(out["loguniform"], 1e-4) self.assertLess(out["loguniform"], 1e-2) self.assertGreaterEqual(out["qloguniform"], 1e-4) self.assertLessEqual(out["qloguniform"], 1e-1) self.assertAlmostEqual(out["qloguniform"] / 5e-5, round(out["qloguniform"] / 5e-5)) self.assertIn(out["choice"], [2, 3, 4]) self.assertGreaterEqual(out["randint"], -9) self.assertLess(out["randint"], 15) self.assertGreaterEqual(out["qrandint"], -21) self.assertLessEqual(out["qrandint"], 12) self.assertEqual(out["qrandint"] % 3, 0) # Very improbable self.assertGreater(out["randn"], 0) self.assertLess(out["randn"], 20) self.assertGreater(out["qrandn"], 0) self.assertLess(out["qrandn"], 20) self.assertAlmostEqual(out["qrandn"] / 0.2, round(out["qrandn"] / 0.2))
def qloguniform(lower, upper, q, base=10): ''' Sample a float between lower and upper. Power distribute uniformly between log_{base}(lower) and log_{base}(upper). Round the result to nearest value with granularity q, include upper. :param lower: Lower bound of the sampling range. :param upper: Upper bound of the sampling range. :param q: Granularity for increment. :param base: Log base for distribution. Default to 10. ''' return tune.qloguniform(lower, upper, q, base)
# __run_tunable_samples_start__ tune.run(trainable, config={"a": 2, "b": 4}, num_samples=10) # __run_tunable_samples_end__ # __search_space_start__ space = {"a": tune.uniform(0, 1), "b": tune.uniform(0, 1)} tune.run(trainable, config=space, num_samples=10) # __search_space_end__ # __config_start__ config = { "uniform": tune.uniform(-5, -1), # Uniform float between -5 and -1 "quniform": tune.quniform(3.2, 5.4, 0.2), # Round to increments of 0.2 "loguniform": tune.loguniform(1e-4, 1e-1), # Uniform float in log space "qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5), # Round to increments of 0.00005 "randn": tune.randn(10, 2), # Normal distribution with mean 10 and sd 2 "qrandn": tune.qrandn(10, 2, 0.2), # Round to increments of 0.2 "randint": tune.randint(-9, 15), # Random integer between -9 and 15 "qrandint": tune.qrandint(-21, 12, 3), # Round to increments of 3 (includes 12) "lograndint": tune.lograndint(1, 10), # Random integer in log space "qlograndint": tune.qlograndint(1, 10, 2), # Round to increments of 2 "choice": tune.choice(["a", "b", "c"]), # Choose one of these options uniformly "func": tune.sample_from( lambda spec: spec.config.uniform * 0.01), # Depends on other value "grid": tune.grid_search([32, 64, 128]), # Search over all these values } # __config_end__
def cifar10_main(method='BlendSearch', num_samples=10, max_num_epochs=100, gpus_per_trial=2): data_dir = os.path.abspath("test/data") load_data(data_dir) # Download data for all trials before starting the run if method == 'BlendSearch': from flaml import tune else: from ray import tune if method in ['BlendSearch', 'BOHB', 'Optuna']: config = { "l1": tune.randint(2, 8), "l2": tune.randint(2, 8), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs, q=1), "batch_size": tune.randint(1, 4) #tune.choice([2, 4, 8, 16]) } else: config = { "l1": tune.randint(2, 9), "l2": tune.randint(2, 9), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1), "batch_size": tune.randint(1, 5) #tune.choice([2, 4, 8, 16]) } import ray time_budget_s = 3600 start_time = time.time() if method == 'BlendSearch': result = tune.run(ray.tune.with_parameters(train_cifar, data_dir=data_dir), init_config={ "l1": 2, "l2": 2, "num_epochs": 1, "batch_size": 4, }, metric="loss", mode="min", max_resource=max_num_epochs, min_resource=1, report_intermediate_result=True, resources_per_trial={ "cpu": 2, "gpu": gpus_per_trial }, config=config, local_dir='logs/', num_samples=num_samples, time_budget_s=time_budget_s, use_ray=True) else: if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB() scheduler = HyperBandForBOHB(max_t=max_num_epochs) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "l1": 2, "l2": 2, "num_epochs": 1, "batch_size": 4, }]) elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1) result = tune.run(tune.with_parameters(train_cifar, data_dir=data_dir), resources_per_trial={ "cpu": 2, "gpu": gpus_per_trial }, config=config, metric="loss", mode="min", num_samples=num_samples, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo) ray.shutdown() logger.info(f"method={method}") logger.info(f"n_samples={num_samples}") logger.info(f"time={time.time()-start_time}") best_trial = result.get_best_trial("loss", "min", "all") logger.info("Best trial config: {}".format(best_trial.config)) logger.info("Best trial final validation loss: {}".format( best_trial.metric_analysis["loss"]["min"])) logger.info("Best trial final validation accuracy: {}".format( best_trial.metric_analysis["accuracy"]["max"])) best_trained_model = Net(2**best_trial.config["l1"], 2**best_trial.config["l2"]) device = "cpu" if torch.cuda.is_available(): device = "cuda:0" if gpus_per_trial > 1: best_trained_model = nn.DataParallel(best_trained_model) best_trained_model.to(device) checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint") model_state, optimizer_state = torch.load(checkpoint_path) best_trained_model.load_state_dict(model_state) test_acc = _test_accuracy(best_trained_model, device) logger.info("Best trial test set accuracy: {}".format(test_acc))
def decode_tuning_param_config(encoded_tuning_params_config): params = json.loads(encoded_tuning_params_config) config = {} for param in params: # if injecting some constant into trainables if isinstance(params[param], int) or isinstance( params[param], str): config[param] = params[param] continue param_data = params[param] if "is_trivial_enum" in param_data and param_data[ "is_trivial_enum"]: config[param] = tune.choice(param_data["values"]) continue d_from = param_data["domain_from"] d_to = param_data["domain_to"] d_step = param_data["domain_step"] if param_data["type"] == "float": if "use_log_scale" in param_data and param_data[ "use_log_scale"]: if d_step == 0: config[param] = tune.loguniform(d_from, d_to) else: config[param] = tune.qloguniform(d_from, d_to, d_step) else: if d_step == 0: config[param] = tune.uniform(d_from, d_to) else: first_val = (d_from // d_step + 1) * d_step iters = int(round((d_to - first_val) / d_step + 1, 1)) param_domain = [d_from] param_domain += [ round(first_val + i * d_step, 3) for i in range(iters) ] config[param] = tune.choice(param_domain) else: if "use_log_scale" in param_data and param_data[ "use_log_scale"]: if d_step == 0: config[param] = tune.lograndint(d_from, d_to + 1) else: # Warning: qlograndint does not actually have uniform distr. config[param] = tune.qlograndint(d_from, d_to, d_step) else: if d_step == 0 or d_step == 1: config[param] = tune.randint(d_from, d_to + 1) else: # RayTune does not provide quantized *uniform* distribution; # hence, manually quantize by using tune.choice instead first_val = (d_from // d_step + 1) * d_step iters = int(round((d_to - first_val) / d_step + 1, 1)) param_domain = [d_from] param_domain += [ first_val + i * d_step for i in range(iters) ] config[param] = tune.choice(param_domain) return config
def cifar10_main(method="BlendSearch", num_samples=10, max_num_epochs=100, gpus_per_trial=1): data_dir = os.path.abspath("test/data") load_data(data_dir) # Download data for all trials before starting the run if method == "BlendSearch": from flaml import tune else: from ray import tune if method in ["BOHB"]: config = { "l1": tune.randint(2, 8), "l2": tune.randint(2, 8), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.qloguniform(1, max_num_epochs, q=1), "batch_size": tune.randint(1, 4), } else: config = { "l1": tune.randint(2, 9), "l2": tune.randint(2, 9), "lr": tune.loguniform(1e-4, 1e-1), "num_epochs": tune.loguniform(1, max_num_epochs), "batch_size": tune.randint(1, 5), } import ray time_budget_s = 600 np.random.seed(7654321) start_time = time.time() if method == "BlendSearch": result = tune.run( ray.tune.with_parameters(train_cifar, data_dir=data_dir), config=config, metric="loss", mode="min", low_cost_partial_config={"num_epochs": 1}, max_resource=max_num_epochs, min_resource=1, scheduler="asha", resources_per_trial={ "cpu": 1, "gpu": gpus_per_trial }, local_dir="logs/", num_samples=num_samples, time_budget_s=time_budget_s, use_ray=True, ) else: if "ASHA" == method: algo = None elif "BOHB" == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB() scheduler = HyperBandForBOHB(max_t=max_num_epochs) elif "Optuna" == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch(seed=10) elif "CFO" == method: from flaml import CFO algo = CFO(low_cost_partial_config={ "num_epochs": 1, }) elif "Nevergrad" == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) if method != "BOHB": from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1) result = tune.run( tune.with_parameters(train_cifar, data_dir=data_dir), resources_per_trial={ "cpu": 1, "gpu": gpus_per_trial }, config=config, metric="loss", mode="min", num_samples=num_samples, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo, ) ray.shutdown() logger.info(f"method={method}") logger.info(f"#trials={len(result.trials)}") logger.info(f"time={time.time()-start_time}") best_trial = result.get_best_trial("loss", "min", "all") logger.info("Best trial config: {}".format(best_trial.config)) logger.info("Best trial final validation loss: {}".format( best_trial.metric_analysis["loss"]["min"])) logger.info("Best trial final validation accuracy: {}".format( best_trial.metric_analysis["accuracy"]["max"])) best_trained_model = Net(2**best_trial.config["l1"], 2**best_trial.config["l2"]) device = "cpu" if torch.cuda.is_available(): device = "cuda:0" if gpus_per_trial > 1: best_trained_model = nn.DataParallel(best_trained_model) best_trained_model.to(device) checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint") model_state, optimizer_state = torch.load(checkpoint_path) best_trained_model.load_state_dict(model_state) test_acc = _test_accuracy(best_trained_model, device) logger.info("Best trial test set accuracy: {}".format(test_acc))
# ------------------------------------------------------------------- # evaluation # ------------------------------------------------------------------- objective = "weighted" search_alg = HyperOptSearch(metric=objective, mode="min") search_alg = ConcurrencyLimiter(search_alg, max_concurrent=4) reporter = CLIReporter(metric_columns=[ "1-nn-accuracy-cd", "mmd-cd", "cov-cd", "jsd", "weighted" ]) analysis = tune.run( tune.with_parameters(evaluation, fakes=fakes, lidar=lidar), config={ "tol": tune.qloguniform(1e-3, 1e-1, 5e-4), "num_points": args.num_points, "batch_size": cfg.solver.batch_size, }, resources_per_trial={ "cpu": 4, "gpu": 0.5 }, num_samples=100, local_dir=osp.join(kwargs["project_dir"], "tol_tuning"), search_alg=search_alg, progress_reporter=reporter, ) best_config = analysis.get_best_config(metric=objective, mode="min") print("Best config: ", best_config)