def update_search_space(self, search_space):
'''
Tuners are advised to support updating search space at run-time.
If a tuner can only set search space once before generating first hyper-parameters,
it should explicitly document this behaviour.
search_space: JSON object created by experiment owner
'''
config = {}
for key, value in search_space:
v = value.get("_value")
_type = value['_type']
if _type == 'choice':
config[key] = choice(v)
elif _type == 'randint':
config[key] = randint(v[0], v[1] - 1)
elif _type == 'uniform':
config[key] = uniform(v[0], v[1])
elif _type == 'quniform':
config[key] = quniform(v[0], v[1], v[2])
elif _type == 'loguniform':
config[key] = loguniform(v[0], v[1])
elif _type == 'qloguniform':
config[key] = qloguniform(v[0], v[1], v[2])
elif _type == 'normal':
config[key] = randn(v[1], v[2])
elif _type == 'qnormal':
config[key] = qrandn(v[1], v[2], v[3])
else:
raise ValueError(
f'unsupported type in search_space {_type}')
self._ls.set_search_properties(None, None, config)
if self._gs is not None:
self._gs.set_search_properties(None, None, config)
self._init_search()
def update_search_space(self, search_space):
"""Required by NNI.
Tuners are advised to support updating search space at run-time.
If a tuner can only set search space once before generating first hyper-parameters,
it should explicitly document this behaviour.
Args:
search_space: JSON object created by experiment owner.
"""
config = {}
for key, value in search_space.items():
v = value.get("_value")
_type = value["_type"]
if _type == "choice":
config[key] = choice(v)
elif _type == "randint":
config[key] = randint(*v)
elif _type == "uniform":
config[key] = uniform(*v)
elif _type == "quniform":
config[key] = quniform(*v)
elif _type == "loguniform":
config[key] = loguniform(*v)
elif _type == "qloguniform":
config[key] = qloguniform(*v)
elif _type == "normal":
config[key] = randn(*v)
elif _type == "qnormal":
config[key] = qrandn(*v)
else:
raise ValueError(f"unsupported type in search_space {_type}")
# low_cost_partial_config is passed to constructor,
# which is before update_search_space() is called
init_config = self._ls.init_config
add_cost_to_space(config, init_config, self._cat_hp_cost)
self._ls = self.LocalSearch(
init_config,
self._ls.metric,
self._mode,
config,
self._ls.resource_attr,
self._ls.min_resource,
self._ls.max_resource,
self._ls.resource_multiple_factor,
cost_attr=self.cost_attr,
seed=self._ls.seed,
)
if self._gs is not None:
self._gs = GlobalSearch(
space=config,
metric=self._metric,
mode=self._mode,
sampler=self._gs._sampler,
)
self._gs.space = config
self._init_search()
def testTuneSampleAPI(self):
config = {
"func": tune.sample_from(lambda spec: spec.config.uniform * 0.01),
"uniform": tune.uniform(-5, -1),
"quniform": tune.quniform(3.2, 5.4, 0.2),
"loguniform": tune.loguniform(1e-4, 1e-2),
"qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5),
"choice": tune.choice([2, 3, 4]),
"randint": tune.randint(-9, 15),
"qrandint": tune.qrandint(-21, 12, 3),
"randn": tune.randn(10, 2),
"qrandn": tune.qrandn(10, 2, 0.2),
}
for _, (_, generated) in zip(
range(1000), generate_variants({
"config": config
})):
out = generated["config"]
self.assertAlmostEqual(out["func"], out["uniform"] * 0.01)
self.assertGreaterEqual(out["uniform"], -5)
self.assertLess(out["uniform"], -1)
self.assertGreaterEqual(out["quniform"], 3.2)
self.assertLessEqual(out["quniform"], 5.4)
self.assertAlmostEqual(out["quniform"] / 0.2,
round(out["quniform"] / 0.2))
self.assertGreaterEqual(out["loguniform"], 1e-4)
self.assertLess(out["loguniform"], 1e-2)
self.assertGreaterEqual(out["qloguniform"], 1e-4)
self.assertLessEqual(out["qloguniform"], 1e-1)
self.assertAlmostEqual(out["qloguniform"] / 5e-5,
round(out["qloguniform"] / 5e-5))
self.assertIn(out["choice"], [2, 3, 4])
self.assertGreaterEqual(out["randint"], -9)
self.assertLess(out["randint"], 15)
self.assertGreaterEqual(out["qrandint"], -21)
self.assertLessEqual(out["qrandint"], 12)
self.assertEqual(out["qrandint"] % 3, 0)
# Very improbable
self.assertGreater(out["randn"], 0)
self.assertLess(out["randn"], 20)
self.assertGreater(out["qrandn"], 0)
self.assertLess(out["qrandn"], 20)
self.assertAlmostEqual(out["qrandn"] / 0.2,
round(out["qrandn"] / 0.2))
def qloguniform(lower, upper, q, base=10):
'''
Sample a float between lower and upper.
Power distribute uniformly between log_{base}(lower) and log_{base}(upper).
Round the result to nearest value with granularity q, include upper.
:param lower: Lower bound of the sampling range.
:param upper: Upper bound of the sampling range.
:param q: Granularity for increment.
:param base: Log base for distribution. Default to 10.
'''
return tune.qloguniform(lower, upper, q, base)
# __run_tunable_samples_start__
tune.run(trainable, config={"a": 2, "b": 4}, num_samples=10)
# __run_tunable_samples_end__
# __search_space_start__
space = {"a": tune.uniform(0, 1), "b": tune.uniform(0, 1)}
tune.run(trainable, config=space, num_samples=10)
# __search_space_end__
# __config_start__
config = {
"uniform": tune.uniform(-5, -1), # Uniform float between -5 and -1
"quniform": tune.quniform(3.2, 5.4, 0.2), # Round to increments of 0.2
"loguniform": tune.loguniform(1e-4, 1e-1), # Uniform float in log space
"qloguniform": tune.qloguniform(1e-4, 1e-1,
5e-5), # Round to increments of 0.00005
"randn": tune.randn(10, 2), # Normal distribution with mean 10 and sd 2
"qrandn": tune.qrandn(10, 2, 0.2), # Round to increments of 0.2
"randint": tune.randint(-9, 15), # Random integer between -9 and 15
"qrandint": tune.qrandint(-21, 12,
3), # Round to increments of 3 (includes 12)
"lograndint": tune.lograndint(1, 10), # Random integer in log space
"qlograndint": tune.qlograndint(1, 10, 2), # Round to increments of 2
"choice": tune.choice(["a", "b",
"c"]), # Choose one of these options uniformly
"func": tune.sample_from(
lambda spec: spec.config.uniform * 0.01), # Depends on other value
"grid": tune.grid_search([32, 64, 128]), # Search over all these values
}
# __config_end__
def cifar10_main(method='BlendSearch',
num_samples=10,
max_num_epochs=100,
gpus_per_trial=2):
data_dir = os.path.abspath("test/data")
load_data(data_dir) # Download data for all trials before starting the run
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
if method in ['BlendSearch', 'BOHB', 'Optuna']:
config = {
"l1": tune.randint(2, 8),
"l2": tune.randint(2, 8),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs, q=1),
"batch_size": tune.randint(1, 4) #tune.choice([2, 4, 8, 16])
}
else:
config = {
"l1": tune.randint(2, 9),
"l2": tune.randint(2, 9),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1),
"batch_size": tune.randint(1, 5) #tune.choice([2, 4, 8, 16])
}
import ray
time_budget_s = 3600
start_time = time.time()
if method == 'BlendSearch':
result = tune.run(ray.tune.with_parameters(train_cifar,
data_dir=data_dir),
init_config={
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
},
metric="loss",
mode="min",
max_resource=max_num_epochs,
min_resource=1,
report_intermediate_result=True,
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB()
scheduler = HyperBandForBOHB(max_t=max_num_epochs)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
}])
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1)
result = tune.run(tune.with_parameters(train_cifar, data_dir=data_dir),
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
metric="loss",
mode="min",
num_samples=num_samples,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples}")
logger.info(f"time={time.time()-start_time}")
best_trial = result.get_best_trial("loss", "min", "all")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation loss: {}".format(
best_trial.metric_analysis["loss"]["min"]))
logger.info("Best trial final validation accuracy: {}".format(
best_trial.metric_analysis["accuracy"]["max"]))
best_trained_model = Net(2**best_trial.config["l1"],
2**best_trial.config["l2"])
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
if gpus_per_trial > 1:
best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint_path)
best_trained_model.load_state_dict(model_state)
test_acc = _test_accuracy(best_trained_model, device)
logger.info("Best trial test set accuracy: {}".format(test_acc))
def decode_tuning_param_config(encoded_tuning_params_config):
params = json.loads(encoded_tuning_params_config)
config = {}
for param in params:
# if injecting some constant into trainables
if isinstance(params[param], int) or isinstance(
params[param], str):
config[param] = params[param]
continue
param_data = params[param]
if "is_trivial_enum" in param_data and param_data[
"is_trivial_enum"]:
config[param] = tune.choice(param_data["values"])
continue
d_from = param_data["domain_from"]
d_to = param_data["domain_to"]
d_step = param_data["domain_step"]
if param_data["type"] == "float":
if "use_log_scale" in param_data and param_data[
"use_log_scale"]:
if d_step == 0:
config[param] = tune.loguniform(d_from, d_to)
else:
config[param] = tune.qloguniform(d_from, d_to, d_step)
else:
if d_step == 0:
config[param] = tune.uniform(d_from, d_to)
else:
first_val = (d_from // d_step + 1) * d_step
iters = int(round((d_to - first_val) / d_step + 1, 1))
param_domain = [d_from]
param_domain += [
round(first_val + i * d_step, 3)
for i in range(iters)
]
config[param] = tune.choice(param_domain)
else:
if "use_log_scale" in param_data and param_data[
"use_log_scale"]:
if d_step == 0:
config[param] = tune.lograndint(d_from, d_to + 1)
else:
# Warning: qlograndint does not actually have uniform distr.
config[param] = tune.qlograndint(d_from, d_to, d_step)
else:
if d_step == 0 or d_step == 1:
config[param] = tune.randint(d_from, d_to + 1)
else:
# RayTune does not provide quantized *uniform* distribution;
# hence, manually quantize by using tune.choice instead
first_val = (d_from // d_step + 1) * d_step
iters = int(round((d_to - first_val) / d_step + 1, 1))
param_domain = [d_from]
param_domain += [
first_val + i * d_step for i in range(iters)
]
config[param] = tune.choice(param_domain)
return config
def cifar10_main(method="BlendSearch",
num_samples=10,
max_num_epochs=100,
gpus_per_trial=1):
data_dir = os.path.abspath("test/data")
load_data(data_dir) # Download data for all trials before starting the run
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
if method in ["BOHB"]:
config = {
"l1": tune.randint(2, 8),
"l2": tune.randint(2, 8),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs, q=1),
"batch_size": tune.randint(1, 4),
}
else:
config = {
"l1": tune.randint(2, 9),
"l2": tune.randint(2, 9),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.loguniform(1, max_num_epochs),
"batch_size": tune.randint(1, 5),
}
import ray
time_budget_s = 600
np.random.seed(7654321)
start_time = time.time()
if method == "BlendSearch":
result = tune.run(
ray.tune.with_parameters(train_cifar, data_dir=data_dir),
config=config,
metric="loss",
mode="min",
low_cost_partial_config={"num_epochs": 1},
max_resource=max_num_epochs,
min_resource=1,
scheduler="asha",
resources_per_trial={
"cpu": 1,
"gpu": gpus_per_trial
},
local_dir="logs/",
num_samples=num_samples,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB()
scheduler = HyperBandForBOHB(max_t=max_num_epochs)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch(seed=10)
elif "CFO" == method:
from flaml import CFO
algo = CFO(low_cost_partial_config={
"num_epochs": 1,
})
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1)
result = tune.run(
tune.with_parameters(train_cifar, data_dir=data_dir),
resources_per_trial={
"cpu": 1,
"gpu": gpus_per_trial
},
config=config,
metric="loss",
mode="min",
num_samples=num_samples,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
ray.shutdown()
logger.info(f"method={method}")
logger.info(f"#trials={len(result.trials)}")
logger.info(f"time={time.time()-start_time}")
best_trial = result.get_best_trial("loss", "min", "all")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation loss: {}".format(
best_trial.metric_analysis["loss"]["min"]))
logger.info("Best trial final validation accuracy: {}".format(
best_trial.metric_analysis["accuracy"]["max"]))
best_trained_model = Net(2**best_trial.config["l1"],
2**best_trial.config["l2"])
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
if gpus_per_trial > 1:
best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint_path)
best_trained_model.load_state_dict(model_state)
test_acc = _test_accuracy(best_trained_model, device)
logger.info("Best trial test set accuracy: {}".format(test_acc))
# -------------------------------------------------------------------
# evaluation
# -------------------------------------------------------------------
objective = "weighted"
search_alg = HyperOptSearch(metric=objective, mode="min")
search_alg = ConcurrencyLimiter(search_alg, max_concurrent=4)
reporter = CLIReporter(metric_columns=[
"1-nn-accuracy-cd", "mmd-cd", "cov-cd", "jsd", "weighted"
])
analysis = tune.run(
tune.with_parameters(evaluation, fakes=fakes, lidar=lidar),
config={
"tol": tune.qloguniform(1e-3, 1e-1, 5e-4),
"num_points": args.num_points,
"batch_size": cfg.solver.batch_size,
},
resources_per_trial={
"cpu": 4,
"gpu": 0.5
},
num_samples=100,
local_dir=osp.join(kwargs["project_dir"], "tol_tuning"),
search_alg=search_alg,
progress_reporter=reporter,
)
best_config = analysis.get_best_config(metric=objective, mode="min")
print("Best config: ", best_config)
