def testBOHB(self):
from ray.tune.suggest.bohb import TuneBOHB
searcher = TuneBOHB(space=self.config, metric=self.metric_name, mode="max")
self._save(searcher)
searcher = TuneBOHB(space=self.config, metric=self.metric_name, mode="max")
self._restore(searcher)
def tunerTrain():
ray.init(_memory=4000000000, num_cpus=5)
searchSpace = {
'lr': tune.loguniform(1e-4, 9e-1),
'finalOutput':
tune.uniform(2, 50), # minimum of 2, other 1//2 = 0 activation maps
'stride1': tune.uniform(1, 4),
'stride2': tune.uniform(1, 4),
'batchSize': tune.uniform(2, 32),
'finalChannel': tune.uniform(1, 50),
}
# analysis = tune.run(train, num_samples=1, scheduler=ASHAScheduler(metric='score', mode='max'),
# config=searchSpace)
algo = TuneBOHB(max_concurrent=4, metric="mean_loss", mode="max")
bohb = HyperBandForBOHB(
metric="mean_loss",
mode="max",
)
analysis = tune.run(train,
config=searchSpace,
scheduler=bohb,
search_alg=algo,
num_samples=10)
# bayesopt = BayesOptSearch( metric="mean_loss", mode="max", random_search_steps = 3)
# tune.run(train, search_alg=bayesopt, config= searchSpace, scheduler=ASHAScheduler("mean_loss","max"))
print("Best config: ",
analysis.get_best_config(metric="mean_loss", mode="max"))
def _get_search_algorithm(
self, search_algorithm, config_space, metric, mode, max_concurrent):
if search_algorithm == "BO":
algo = BayesOptSearch(
utility_kwargs={
"kind": "ucb",
"kappa": 2.5,
"xi": 0.0
})
algo = ConcurrencyLimiter(algo, max_concurrent=max_concurrent)
scheduler = AsyncHyperBandScheduler()
elif search_algorithm == "BOHB":
experiment_metrics = dict(metric=metric, mode=mode)
algo = TuneBOHB(
config_space, max_concurrent=max_concurrent, **experiment_metrics)
scheduler = HyperBandForBOHB(
time_attr="training_iteration",
reduction_factor=4)
elif search_algorithm == "PBT":
# Problem of PBT: It mutates the param value, so sometimes, it generates unacceptable values
algo = None
scheduler = PopulationBasedTraining(
time_attr='training_iteration',
perturbation_interval=2, # Every N time_attr units, "perturb" the parameters.
hyperparam_mutations=config_space)
elif search_algorithm == "GRID" or search_algorithm == "RANDOM":
algo = None
scheduler = None
else:
raise Exception(search_algorithm, "is not available yet")
return algo, scheduler
def testBOHB(self):
from ray.tune.suggest.bohb import TuneBOHB
converted_config = TuneBOHB.convert_search_space(self.config)
converted_config.seed(1000) # At least one nan, inf, -inf and float
out = tune.run(_invalid_objective,
search_alg=TuneBOHB(space=converted_config,
metric="_metric",
mode="max"),
metric="_metric",
mode="max",
num_samples=8,
reuse_actors=False)
best_trial = out.best_trial
self.assertLessEqual(best_trial.config["report"], 2.0)
def main(args):
cfg = setup(args)
search_space = CS.ConfigurationSpace()
search_space.add_hyperparameters([
CS.UniformFloatHyperparameter(name="lr", lower=1e-6, upper=1e-3),
CS.UniformFloatHyperparameter(name="wd", lower=0, upper=1e-3),
CS.UniformFloatHyperparameter(name="wd_bias", lower=0, upper=1e-3),
CS.CategoricalHyperparameter(name="bsz",
choices=[64, 96, 128, 160, 224, 256]),
CS.CategoricalHyperparameter(name="num_inst",
choices=[2, 4, 8, 16, 32]),
CS.UniformIntegerHyperparameter(name="delay_iters", lower=20,
upper=60),
CS.UniformFloatHyperparameter(name="ce_scale", lower=0.1, upper=1.0),
CS.UniformIntegerHyperparameter(name="circle_scale",
lower=8,
upper=256),
CS.UniformFloatHyperparameter(name="circle_margin",
lower=0.1,
upper=0.5),
CS.CategoricalHyperparameter(name="autoaug_enabled",
choices=[True, False]),
CS.CategoricalHyperparameter(name="cj_enabled", choices=[True, False]),
])
exp_metrics = dict(metric="score", mode="max")
bohb_hyperband = HyperBandForBOHB(
time_attr="training_iteration",
max_t=7,
**exp_metrics,
)
bohb_search = TuneBOHB(search_space, max_concurrent=4, **exp_metrics)
reporter = CLIReporter(parameter_columns=["bsz", "num_inst", "lr"],
metric_columns=["r1", "map", "training_iteration"])
analysis = tune.run(partial(train_reid_tune, cfg),
resources_per_trial={
"cpu": 10,
"gpu": 1
},
search_alg=bohb_search,
num_samples=args.num_samples,
scheduler=bohb_hyperband,
progress_reporter=reporter,
local_dir=cfg.OUTPUT_DIR,
keep_checkpoints_num=4,
name="bohb")
best_trial = analysis.get_best_trial("map", "max", "last")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation mAP: {}, Rank-1: {}".format(
best_trial.last_result["map"], best_trial.last_result["r1"]))
def testBOHB(self):
from ray.tune.suggest.bohb import TuneBOHB
out = tune.run(_invalid_objective,
search_alg=TuneBOHB(seed=1000),
config=self.config,
mode="max",
num_samples=8,
reuse_actors=False)
best_trial = out.best_trial
self.assertLessEqual(best_trial.config["report"], 2.0)
def testConvertBOHB(self):
from ray.tune.suggest.bohb import TuneBOHB
import ConfigSpace
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = TuneBOHB.convert_search_space(config)
bohb_config = ConfigSpace.ConfigurationSpace()
bohb_config.add_hyperparameters([
ConfigSpace.CategoricalHyperparameter("a", [2, 3, 4]),
ConfigSpace.UniformIntegerHyperparameter("b/x",
lower=0,
upper=4,
q=2),
ConfigSpace.UniformFloatHyperparameter("b/z",
lower=1e-4,
upper=1e-2,
log=True)
])
converted_config.seed(1234)
bohb_config.seed(1234)
searcher1 = TuneBOHB(space=converted_config)
searcher2 = TuneBOHB(space=bohb_config)
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = TuneBOHB(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["a"], [2, 3, 4])
self.assertEqual(trial.config["b"]["y"], 4)
def set_basic_conf(self):
space = {
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100)
}
def cost(space, reporter):
for i in range(10):
reporter(loss=(space["height"] - 14)**2 -
abs(space["width"] - 3 - i))
search_alg = TuneBOHB(space=space, metric="loss", mode="min", seed=1)
return search_alg, cost
def setup_tune_scheduler():
config_space = workload.create_ch()
experiment_metrics = workload.exp_metric()
bohb_hyperband = HyperBandForBOHB(time_attr="training_iteration",
max_t=243,
reduction_factor=3,
**experiment_metrics)
bohb_search = TuneBOHB(config_space, **experiment_metrics)
return dict(
scheduler=bohb_hyperband,
search_alg=bohb_search,
resources_per_trial=com.detect_baseline_resource(),
)
def setup_tune_scheduler(num_worker):
# BOHB uses ConfigSpace for their hyperparameter search space
config_space = workload.create_ch()
experiment_metrics = workload.exp_metric()
bohb_search = TuneBOHB(config_space, **experiment_metrics)
bohb_hyperband = SyncBOHB(time_attr="training_iteration",
max_t=81,
reduction_factor=3,
**experiment_metrics)
return dict(
scheduler=bohb_hyperband,
search_alg=bohb_search,
resources_per_trial=com.detect_baseline_resource(),
)
# CS.UniformFloatHyperparameter("width", lower=0, upper=20))
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
# config_space.add_hyperparameter(
# CS.CategoricalHyperparameter(
# "activation", choices=["relu", "tanh"]))
bohb_hyperband = HyperBandForBOHB(
time_attr="training_iteration",
max_t=100,
reduction_factor=4,
stop_last_trials=False,
)
bohb_search = TuneBOHB(
# space=config_space, # If you want to set the space manually
)
bohb_search = tune.suggest.ConcurrencyLimiter(bohb_search, max_concurrent=4)
analysis = tune.run(
MyTrainableClass,
name="bohb_test",
config=config,
scheduler=bohb_hyperband,
search_alg=bohb_search,
num_samples=10,
stop={"training_iteration": 100},
metric="episode_reward_mean",
mode="max",
)
# losses.ACLower(),
# losses.Motication(),
# losses.GaussACDC(),
],
))
add(cat("max_validation_steps", [100]))
add(cat("batches_per_step", [10]))
metric = dict(metric="validation_loss", mode="min")
scheduling = HyperBandForBOHB(time_attr="training_iteration",
max_t=500,
reduction_factor=3,
**metric)
search = TuneBOHB(config_space, **metric)
if __name__ == "__main__":
#sys.exit(10)
# ray.init(local_mode=True) # use only one thread to make debugging easier
analysis = tune.run(
trainable.MultipackTrainable,
name="MPT_long_run",
scheduler=scheduling,
search_alg=search,
num_samples=1000,
stop={"training_iteration": 500},
max_failures=0, # for debugging purposes
resources_per_trial={
"cpu": CPU_WORKERS_PER_TRAIL,
"gpu": 1
])
# Feed the score back back to Tune.
del fx_model
gc.collect()
tune.report(total_score=score)
config = {
"top_p": tune.uniform(0.80, 0.999),
"hidden_size": tune.randint(128, 512)
}
bohb_hyperband = HyperBandForBOHB(time_attr="training_iteration",
max_t=100,
reduction_factor=2)
bohb_search = TuneBOHB(max_concurrent=2)
analysis = tune.run(training_function,
name="bohb_test",
config=config,
scheduler=bohb_hyperband,
search_alg=bohb_search,
num_samples=12,
metric="total_score",
mode="max",
resources_per_trial={'gpu': 1},
callbacks=[CSVLoggerCallback(),
JsonLoggerCallback()])
print("Best hyperparameters found were: ", analysis.best_config)
def cifar10_main(method="BlendSearch",
num_samples=10,
max_num_epochs=100,
gpus_per_trial=1):
data_dir = os.path.abspath("test/data")
load_data(data_dir) # Download data for all trials before starting the run
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
if method in ["BOHB"]:
config = {
"l1": tune.randint(2, 8),
"l2": tune.randint(2, 8),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs, q=1),
"batch_size": tune.randint(1, 4),
}
else:
config = {
"l1": tune.randint(2, 9),
"l2": tune.randint(2, 9),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.loguniform(1, max_num_epochs),
"batch_size": tune.randint(1, 5),
}
import ray
time_budget_s = 600
np.random.seed(7654321)
start_time = time.time()
if method == "BlendSearch":
result = tune.run(
ray.tune.with_parameters(train_cifar, data_dir=data_dir),
config=config,
metric="loss",
mode="min",
low_cost_partial_config={"num_epochs": 1},
max_resource=max_num_epochs,
min_resource=1,
scheduler="asha",
resources_per_trial={
"cpu": 1,
"gpu": gpus_per_trial
},
local_dir="logs/",
num_samples=num_samples,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB()
scheduler = HyperBandForBOHB(max_t=max_num_epochs)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch(seed=10)
elif "CFO" == method:
from flaml import CFO
algo = CFO(low_cost_partial_config={
"num_epochs": 1,
})
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1)
result = tune.run(
tune.with_parameters(train_cifar, data_dir=data_dir),
resources_per_trial={
"cpu": 1,
"gpu": gpus_per_trial
},
config=config,
metric="loss",
mode="min",
num_samples=num_samples,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
ray.shutdown()
logger.info(f"method={method}")
logger.info(f"#trials={len(result.trials)}")
logger.info(f"time={time.time()-start_time}")
best_trial = result.get_best_trial("loss", "min", "all")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation loss: {}".format(
best_trial.metric_analysis["loss"]["min"]))
logger.info("Best trial final validation accuracy: {}".format(
best_trial.metric_analysis["accuracy"]["max"]))
best_trained_model = Net(2**best_trial.config["l1"],
2**best_trial.config["l2"])
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
if gpus_per_trial > 1:
best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint_path)
best_trained_model.load_state_dict(model_state)
test_acc = _test_accuracy(best_trained_model, device)
logger.info("Best trial test set accuracy: {}".format(test_acc))
with open(checkpoint_path) as f:
self.timestep = json.loads(f.read())["timestep"]
if __name__ == "__main__":
import ConfigSpace as CS
ray.init(redis_address=args.ray_redis_address)
# BOHB uses ConfigSpace for their hyperparameter search space
config_space = CS.ConfigurationSpace()
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter("height", lower=10, upper=100))
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter("width", lower=0, upper=100))
experiment_metrics = dict(metric="episode_reward_mean", mode="max")
bohb_hyperband = HyperBandForBOHB(time_attr="training_iteration",
max_t=100,
reduction_factor=4,
**experiment_metrics)
bohb_search = TuneBOHB(config_space,
max_concurrent=4,
**experiment_metrics)
run(MyTrainableClass,
name="bohb_test",
scheduler=bohb_hyperband,
search_alg=bohb_search,
num_samples=10,
stop={"training_iteration": 10 if args.smoke_test else 100})
def main(args):
cfg = setup(args)
exp_metrics = dict(metric="score", mode="max")
if args.srch_algo == "hyperopt":
# Create a HyperOpt search space
search_space = {
# "lr": hp.loguniform("lr", np.log(1e-6), np.log(1e-3)),
# "delay_epochs": hp.randint("delay_epochs", 20, 60),
# "wd": hp.uniform("wd", 0, 1e-3),
# "wd_bias": hp.uniform("wd_bias", 0, 1e-3),
"bsz": hp.choice("bsz", [64, 96, 128, 160, 224, 256]),
"num_inst": hp.choice("num_inst", [2, 4, 8, 16, 32]),
# "ce_scale": hp.uniform("ce_scale", 0.1, 1.0),
# "circle_scale": hp.choice("circle_scale", [16, 32, 64, 128, 256]),
# "circle_margin": hp.uniform("circle_margin", 0, 1) * 0.4 + 0.1,
}
current_best_params = [{
"bsz": 0, # index of hp.choice list
"num_inst": 3,
}]
search_algo = HyperOptSearch(search_space,
points_to_evaluate=current_best_params,
**exp_metrics)
if args.pbt:
scheduler = PopulationBasedTraining(
time_attr="training_iteration",
**exp_metrics,
perturbation_interval=2,
hyperparam_mutations={
"bsz": [64, 96, 128, 160, 224, 256],
"num_inst": [2, 4, 8, 16, 32],
})
else:
scheduler = ASHAScheduler(grace_period=2,
reduction_factor=3,
max_t=7,
**exp_metrics)
elif args.srch_algo == "bohb":
search_space = CS.ConfigurationSpace()
search_space.add_hyperparameters([
# CS.UniformFloatHyperparameter(name="lr", lower=1e-6, upper=1e-3, log=True),
# CS.UniformIntegerHyperparameter(name="delay_epochs", lower=20, upper=60),
# CS.UniformFloatHyperparameter(name="ce_scale", lower=0.1, upper=1.0),
# CS.UniformIntegerHyperparameter(name="circle_scale", lower=8, upper=128),
# CS.UniformFloatHyperparameter(name="circle_margin", lower=0.1, upper=0.5),
# CS.UniformFloatHyperparameter(name="wd", lower=0, upper=1e-3),
# CS.UniformFloatHyperparameter(name="wd_bias", lower=0, upper=1e-3),
CS.CategoricalHyperparameter(name="bsz",
choices=[64, 96, 128, 160, 224, 256]),
CS.CategoricalHyperparameter(name="num_inst",
choices=[2, 4, 8, 16, 32]),
# CS.CategoricalHyperparameter(name="autoaug_enabled", choices=[True, False]),
# CS.CategoricalHyperparameter(name="cj_enabled", choices=[True, False]),
])
search_algo = TuneBOHB(search_space, max_concurrent=4, **exp_metrics)
scheduler = HyperBandForBOHB(
time_attr="training_iteration",
reduction_factor=3,
max_t=7,
**exp_metrics,
)
else:
raise ValueError(
"Search algorithm must be chosen from [hyperopt, bohb], but got {}"
.format(args.srch_algo))
reporter = CLIReporter(parameter_columns=["bsz", "num_inst"],
metric_columns=["r1", "map", "training_iteration"])
analysis = tune.run(partial(train_tuner, cfg=cfg),
resources_per_trial={
"cpu": 4,
"gpu": 1
},
search_alg=search_algo,
num_samples=args.num_trials,
scheduler=scheduler,
progress_reporter=reporter,
local_dir=cfg.OUTPUT_DIR,
keep_checkpoints_num=10,
name=args.srch_algo)
best_trial = analysis.get_best_trial("score", "max", "last")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation mAP: {}, Rank-1: {}".format(
best_trial.last_result["map"], best_trial.last_result["r1"]))
save_dict = dict(R1=best_trial.last_result["r1"].item(),
mAP=best_trial.last_result["map"].item())
save_dict.update(best_trial.config)
path = os.path.join(cfg.OUTPUT_DIR, "best_config.yaml")
with PathManager.open(path, "w") as f:
f.write(CfgNode(save_dict).dump())
logger.info("Best config saved to {}".format(os.path.abspath(path)))
def cifar10_main(method='BlendSearch',
num_samples=10,
max_num_epochs=100,
gpus_per_trial=2):
data_dir = os.path.abspath("test/data")
load_data(data_dir) # Download data for all trials before starting the run
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
if method in ['BlendSearch', 'BOHB', 'Optuna']:
config = {
"l1": tune.randint(2, 8),
"l2": tune.randint(2, 8),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs, q=1),
"batch_size": tune.randint(1, 4) #tune.choice([2, 4, 8, 16])
}
else:
config = {
"l1": tune.randint(2, 9),
"l2": tune.randint(2, 9),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1),
"batch_size": tune.randint(1, 5) #tune.choice([2, 4, 8, 16])
}
import ray
time_budget_s = 3600
start_time = time.time()
if method == 'BlendSearch':
result = tune.run(ray.tune.with_parameters(train_cifar,
data_dir=data_dir),
init_config={
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
},
metric="loss",
mode="min",
max_resource=max_num_epochs,
min_resource=1,
report_intermediate_result=True,
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB()
scheduler = HyperBandForBOHB(max_t=max_num_epochs)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
}])
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1)
result = tune.run(tune.with_parameters(train_cifar, data_dir=data_dir),
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
metric="loss",
mode="min",
num_samples=num_samples,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples}")
logger.info(f"time={time.time()-start_time}")
best_trial = result.get_best_trial("loss", "min", "all")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation loss: {}".format(
best_trial.metric_analysis["loss"]["min"]))
logger.info("Best trial final validation accuracy: {}".format(
best_trial.metric_analysis["accuracy"]["max"]))
best_trained_model = Net(2**best_trial.config["l1"],
2**best_trial.config["l2"])
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
if gpus_per_trial > 1:
best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint_path)
best_trained_model.load_state_dict(model_state)
test_acc = _test_accuracy(best_trained_model, device)
logger.info("Best trial test set accuracy: {}".format(test_acc))
def get_raytune_search_alg(raytune_cfg, seeds=False):
if (raytune_cfg["sched"] == "pbt") or (raytune_cfg["sched"] == "pb2"):
if raytune_cfg["search_alg"] is not None:
print(
"INFO: Using schedule '{}' is not compatible with Ray Tune search algorithms."
.format(raytune_cfg["sched"]))
print(
"INFO: Uing the Ray Tune {} scheduler without search algorithm"
.format(raytune_cfg["sched"]))
return None
if (raytune_cfg["sched"] == "bohb") or (raytune_cfg["sched"] == "BOHB"):
print(
"INFO: Using TuneBOHB search algorithm since it is required for BOHB shedule"
)
if seeds:
seed = 1234
else:
seed = None
return TuneBOHB(metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
seed=seed)
# requires pip install bayesian-optimization
if raytune_cfg["search_alg"] == "bayes":
print("INFO: Using BayesOptSearch")
return BayesOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
random_search_steps=raytune_cfg["bayes"]["n_random_steps"],
)
# requires pip install hyperopt
if raytune_cfg["search_alg"] == "hyperopt":
print("INFO: Using HyperOptSearch")
return HyperOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
n_initial_points=raytune_cfg["hyperopt"]["n_random_steps"],
# points_to_evaluate=,
)
if raytune_cfg["search_alg"] == "scikit":
print("INFO: Using bayesian optimization from scikit-learn")
return SkOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
convert_to_python=True,
)
if raytune_cfg["search_alg"] == "nevergrad":
print("INFO: Using bayesian optimization from nevergrad")
return NevergradSearch(
optimizer=ng.optimizers.BayesOptim(
pca=False,
init_budget=raytune_cfg["nevergrad"]["n_random_steps"]),
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
)
# HEBO is not yet supported
# if (raytune_cfg["search_alg"] == "hebo") or (raytune_cfg["search_alg"] == "HEBO"):
# print("Using HEBOSearch")
# return HEBOSearch(
# metric=raytune_cfg["default_metric"],
# mode=raytune_cfg["default_mode"],
# # max_concurrent=8,
# )
else:
print("INFO: Not using any Ray Tune search algorithm")
return None
def main(args):
def trainable(config):
print('begin a trial')
args.params = tools.AttrDict(yaml.safe_load(args.params.replace('#', ',')))
args.logdir = args.logdir and os.path.expanduser(args.logdir)
print('debug ', config["divergence_scale"], config["reward_loss_scale"])
with args.params.unlocked:
args.params.divergence_scale = config["divergence_scale"]
args.params.reward_loss_scale = config["reward_loss_scale"]
# args.params.main_learning_rate = config["main_learning_rate"]
args.params.test_steps = 50
# args.params.num_units = config['num_units']
args.params.test_traj = 5
training.utility.set_up_logging()
experiment = training.Experiment(
args.logdir,
process_fn=functools.partial(process, args=args),
num_runs=args.num_runs,
ping_every=args.ping_every,
resume_runs=args.resume_runs)
for run in experiment:
for test_score in run:
if test_score > 1.0:
tune.report(mean_score=test_score)
break
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
# search = {
# "divergence_scale": tune.quniform(1, 30, 1),
# "reward_loss_scale": tune.quniform(1, 50, 1),
# }
search = {
"divergence_scale": tune.grid_search([0.1, 1, 2, 3, 5, 10]),
"reward_loss_scale": tune.grid_search([1, 2, 5, 10, 20]),
}
config_space = CS.ConfigurationSpace(seed=1234)
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name="divergence_scale", lower=1, upper=30))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name="reward_loss_scale", lower=1, upper=50))
# config_space.add_hyperparameter(
# CSH.UniformFloatHyperparameter("main_learning_rate", lower=0.0001, upper=0.05, log=True))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter("main_learning_rate", lower=1, upper=500, log=True))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name="num_units", lower=150, upper=400, q=50))
bayesopt = BayesOptSearch(metric="mean_loss", mode="min")
bohb_hyperband = HyperBandForBOHB(metric="mean_score", mode="max", time_attr="training_iteration", max_t=30,
reduction_factor=3)
bohb_search = TuneBOHB(space=config_space, max_concurrent=1, metric="mean_score", mode="max")
bayesopt = BayesOptSearch(max_concurrent=3, metric="mean_score", mode="max")
asha = ASHAScheduler(metric="mean_score", mode="max", grace_period=6, reduction_factor=3)
analysis = tune.run(
trainable,
config=search,
num_samples=3,
scheduler=asha,
resources_per_trial={"cpu": 16, "gpu": 1},
stop={"training_iteration": 13},
# search_alg=bayesopt,
log_to_file=True
)
df = analysis.results_df
print("Best config: ", analysis.get_best_config(
metric="mean_score", mode="min"))
print(df)
def _tune_run(self, config, resources_per_trial):
"""Wrapper to call ``tune.run``. Multiple estimators are generated when
early stopping is possible, whereas a single estimator is
generated when early stopping is not possible.
Args:
config (dict): Configurations such as hyperparameters to run
``tune.run`` on.
resources_per_trial (dict): Resources to use per trial within Ray.
Accepted keys are `cpu`, `gpu` and custom resources, and values
are integers specifying the number of each resource to use.
Returns:
analysis (`ExperimentAnalysis`): Object returned by
`tune.run`.
"""
stop_condition = {"training_iteration": self.max_iters}
if self.early_stopping is not None:
config["estimator_list"] = [
clone(self.estimator) for _ in range(self.n_splits)
]
if hasattr(self.early_stopping, "_max_t_attr"):
# we want to delegate stopping to schedulers which
# support it, but we want it to stop eventually, just in case
# the solution is to make the stop condition very big
stop_condition = {"training_iteration": self.max_iters * 10}
else:
config["estimator_list"] = [self.estimator]
if self.search_optimization == "random":
run_args = dict(scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stop_condition,
num_samples=self.num_samples,
config=config,
fail_fast=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
if isinstance(self.param_distributions, list):
run_args["search_alg"] = RandomListSearcher(
self.param_distributions)
analysis = tune.run(_Trainable, **run_args)
return analysis
elif self.search_optimization == "bayesian":
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
hyperparameter_names, spaces = self._get_skopt_params()
search_algo = SkOptSearch(Optimizer(spaces),
hyperparameter_names,
metric="average_test_score",
**self.search_kwargs)
elif self.search_optimization == "bohb":
from ray.tune.suggest.bohb import TuneBOHB
config_space = self._get_bohb_config_space()
search_algo = TuneBOHB(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
elif self.search_optimization == "optuna":
from ray.tune.suggest.optuna import OptunaSearch
config_space = self._get_optuna_params()
search_algo = OptunaSearch(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
elif self.search_optimization == "hyperopt":
from ray.tune.suggest.hyperopt import HyperOptSearch
config_space = self._get_hyperopt_params()
search_algo = HyperOptSearch(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
if isinstance(self.n_jobs, int) and self.n_jobs > 0:
search_algo = ConcurrencyLimiter(search_algo,
max_concurrent=self.n_jobs)
analysis = tune.run(_Trainable,
search_alg=search_algo,
scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stop_condition,
num_samples=self.num_samples,
config=config,
fail_fast=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
return analysis
# gin-config
gin.parse_config_files_and_bindings(['configs/config.gin'], bindings)
utils_params.save_config(run_paths['path_gin'], gin.config_str())
# setup pipeline
ds_train, ds_val, ds_test = load_from_tfrecords()
# model
model = TransformerS2S()
trainer = Trainer(model, ds_train, ds_val, run_paths)
for val_accuracy in trainer.train():
tune.report(val_accuracy=val_accuracy)
algo = TuneBOHB(max_concurrent=4, metric="val_accuracy", mode="max")
bohb = HyperBandForBOHB(time_attr="training_iteration",
metric="val_accuracy",
mode="max",
max_t=100)
config_name = 'transformerS2S'
if config_name == 'lstm':
config = {
"LSTM.rnn_units1": tune.randint(16, 64),
"LSTM.rnn_units2": tune.randint(8, 32),
"LSTM.dense_units": tune.randint(12, 64),
"LSTM.dropout_rate": tune.uniform(0, 0.8),
"load_from_tfrecords.batch_size": tune.choice([8, 16, 32, 64, 128])
}
def _tune_run(self, config, resources_per_trial):
"""Wrapper to call ``tune.run``. Multiple estimators are generated when
early stopping is possible, whereas a single estimator is
generated when early stopping is not possible.
Args:
config (dict): Configurations such as hyperparameters to run
``tune.run`` on.
resources_per_trial (dict): Resources to use per trial within Ray.
Accepted keys are `cpu`, `gpu` and custom resources, and values
are integers specifying the number of each resource to use.
Returns:
analysis (`ExperimentAnalysis`): Object returned by
`tune.run`.
"""
if self.seed is not None:
random.seed(self.seed)
np.random.seed(self.seed)
trainable = _Trainable
if self.pipeline_auto_early_stop and check_is_pipeline(
self.estimator) and self.early_stopping:
trainable = _PipelineTrainable
max_iter = self.max_iters
if self.early_stopping is not None:
config["estimator_list"] = [
clone(self.estimator) for _ in range(self.n_splits)
]
if hasattr(self.early_stopping, "_max_t_attr"):
# we want to delegate stopping to schedulers which
# support it, but we want it to stop eventually, just in case
# the solution is to make the stop condition very big
max_iter = self.max_iters * 10
else:
config["estimator_list"] = [self.estimator]
stopper = MaximumIterationStopper(max_iter=max_iter)
if self.stopper:
stopper = CombinedStopper(stopper, self.stopper)
run_args = dict(scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stopper,
num_samples=self.n_trials,
config=config,
fail_fast="raise",
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir),
loggers=self.loggers,
time_budget_s=self.time_budget_s)
if self.search_optimization == "random":
if isinstance(self.param_distributions, list):
search_algo = RandomListSearcher(self.param_distributions)
else:
search_algo = BasicVariantGenerator()
run_args["search_alg"] = search_algo
else:
search_space = None
override_search_space = True
if self._is_param_distributions_all_tune_domains():
run_args["config"].update(self.param_distributions)
override_search_space = False
search_kwargs = self.search_kwargs.copy()
search_kwargs.update(metric=self._metric_name, mode="max")
if self.search_optimization == "bayesian":
from ray.tune.suggest.skopt import SkOptSearch
if override_search_space:
search_space = self.param_distributions
search_algo = SkOptSearch(space=search_space, **search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "bohb":
from ray.tune.suggest.bohb import TuneBOHB
if override_search_space:
search_space = self._get_bohb_config_space()
if self.seed:
warnings.warn("'seed' is not implemented for BOHB.")
search_algo = TuneBOHB(space=search_space, **search_kwargs)
# search_algo = TuneBOHB(
# space=search_space, seed=self.seed, **search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "optuna":
from ray.tune.suggest.optuna import OptunaSearch
from optuna.samplers import TPESampler
sampler = TPESampler(seed=self.seed)
if override_search_space:
search_space = self._get_optuna_params()
search_algo = OptunaSearch(space=search_space,
sampler=sampler,
**search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "hyperopt":
from ray.tune.suggest.hyperopt import HyperOptSearch
if override_search_space:
search_space = self._get_hyperopt_params()
search_algo = HyperOptSearch(space=search_space,
random_state_seed=self.seed,
**search_kwargs)
run_args["search_alg"] = search_algo
else:
# This should not happen as we validate the input before
# this method. Still, just to be sure, raise an error here.
raise ValueError(
f"Invalid search optimizer: {self.search_optimization}")
if isinstance(self.n_jobs, int) and self.n_jobs > 0 \
and not self.search_optimization == "random":
search_algo = ConcurrencyLimiter(search_algo,
max_concurrent=self.n_jobs)
run_args["search_alg"] = search_algo
with warnings.catch_warnings():
warnings.filterwarnings("ignore",
message="fail_fast='raise' "
"detected.")
analysis = tune.run(trainable, **run_args)
return analysis
def tune(args):
search_space = get_search_space(args.config_name)
config = get_config(args.config_name)
store = get_target_path(config, 'hptune')
if args.overwrite:
if os.path.isdir(store):
shutil.rmtree(store)
else:
if os.path.isdir(store):
raise ValueError(
f'The tune directory {store} exists. Set flag "--overwrite" '
'if you want to overwrite runs - all existing runs will be lost!')
os.makedirs(store)
config.update({
'is_tune': True,
'small_aoi': args.small_aoi,
})
ngpu = torch.cuda.device_count()
ncpu = os.cpu_count()
max_concurrent = int(
np.min((
np.floor(ncpu / config['ncpu_per_run']),
np.floor(ngpu / config['ngpu_per_run'])
))
)
print(
'\nTuning hyperparameters;\n'
f' Available resources: {ngpu} GPUs | {ncpu} CPUs\n'
f' Number of concurrent runs: {max_concurrent}\n'
)
bobh_search = TuneBOHB(
space=search_space,
max_concurrent=max_concurrent,
metric=config['metric'],
mode='min'
)
bohb_scheduler = HyperBandForBOHB(
time_attr='epoch',
metric=config['metric'],
mode='min',
max_t=config['max_t'],
reduction_factor=config['halving_factor'])
if args.run_single:
logging.warning('Starting test run.')
e = Emulator(search_space.sample_configuration())
logging.warning('Starting training loop.')
e._train()
logging.warning('Finishing test run.')
sys.exit('0')
ray.tune.run(
Emulator,
config={'hc_config': config},
resources_per_trial={
'cpu': config['ncpu_per_run'],
'gpu': config['ngpu_per_run']},
num_samples=config['num_samples'],
local_dir=store,
raise_on_failed_trial=True,
verbose=1,
with_server=False,
ray_auto_init=False,
search_alg=bobh_search,
scheduler=bohb_scheduler,
loggers=[JsonLogger, CSVLogger],
keep_checkpoints_num=1,
reuse_actors=False,
stop={'patience_counter': config['patience']}
)
summarize_run(store, overwrite=True)
def _test_xgboost(method="BlendSearch"):
try:
import ray
except ImportError:
return
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
search_space = {
"max_depth":
tune.randint(1, 9)
if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight":
tune.choice([1, 2, 3]),
"subsample":
tune.uniform(0.5, 1.0),
"eta":
tune.loguniform(1e-4, 1e-1),
}
max_iter = 10
for num_samples in [128]:
time_budget_s = 60
for n_cpu in [2]:
start_time = time.time()
# ray.init(address='auto')
if method == "BlendSearch":
analysis = tune.run(
train_breast_cancer,
config=search_space,
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
scheduler="asha",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif "CFO" == method:
from flaml import CFO
algo = CFO(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "CFOCat" == method:
from flaml.searcher.cfo_cat import CFOCat
algo = CFOCat(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "Dragonfly" == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif "SkOpt" == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif "ZOOpt" == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples * n_cpu)
elif "Ax" == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif "HyperOpt" == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_iter, grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
# # Load the best model checkpoint
# import os
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss", "min", "all")
accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def hparams(algorithm, scheduler, num_samples, tensorboard, bare):
from glob import glob
import tensorflow.summary
from tensorflow import random as tfrandom, int64 as tfint64
from ray import init as init_ray, shutdown as shutdown_ray
from ray import tune
from wandb.ray import WandbLogger
from wandb import sweep as wandbsweep
from wandb.apis import CommError as wandbCommError
# less summaries are logged if MLENCRYPT_TB is TRUE (for efficiency)
# TODO: use tf.summary.record_if?
environ["MLENCRYPT_TB"] = str(tensorboard).upper()
environ["MLENCRYPT_BARE"] = str(bare).upper()
if getenv('MLENCRYPT_TB', 'FALSE') == 'TRUE' and \
getenv('MLENCRYPT_BARE', 'FALSE') == 'TRUE':
raise ValueError('TensorBoard logging cannot be enabled in bare mode.')
logdir = f'logs/hparams/{datetime.now()}'
# "These results show that K = 3 is the optimal choice for the
# cryptographic application of neural synchronization. K = 1 and K = 2 are
# too insecure in regard to the geometric attack. And for K > 3 the effort
# of A and B grows exponentially with increasing L, while the simple attack
# is quite successful in the limit K -> infinity. Consequently, one should
# only use Tree Parity Machines with three hidden units for the neural
# key-exchange protocol." (Ruttor, 2006)
# https://arxiv.org/pdf/0711.2411.pdf#page=59
update_rules = [
'random-same',
# 'random-different-A-B-E', 'random-different-A-B',
'hebbian',
'anti_hebbian',
'random_walk'
]
K_bounds = {'min': 4, 'max': 8}
N_bounds = {'min': 4, 'max': 8}
L_bounds = {'min': 4, 'max': 8}
# TODO: don't use *_bounds.values() since .values doesn't preserve order
def get_session_num(logdir):
current_runs = glob(join(logdir, "run-*"))
if current_runs:
last_run_path = current_runs[-1]
last_run_session_num = int(last_run_path.split('-')[-1])
return last_run_session_num + 1
else: # there are no runs yet, start at 0
return 0
def trainable(config, reporter):
"""
Args:
config (dict): Parameters provided from the search algorithm
or variant generation.
"""
if not isinstance(config['update_rule'], str):
update_rule = update_rules[int(config['update_rule'])]
else:
update_rule = config['update_rule']
K, N, L = int(config['K']), int(config['N']), int(config['L'])
run_name = f"run-{get_session_num(logdir)}"
run_logdir = join(logdir, run_name)
# for each attack, the TPMs should start with the same weights
initial_weights_tensors = get_initial_weights(K, N, L)
training_steps_ls = {}
eve_scores_ls = {}
losses_ls = {}
# for each attack, the TPMs should use the same inputs
seed = tfrandom.uniform([],
minval=0,
maxval=tfint64.max,
dtype=tfint64).numpy()
for attack in ['none', 'geometric']:
initial_weights = {
tpm: weights_tensor_to_variable(weights, tpm)
for tpm, weights in initial_weights_tensors.items()
}
tfrandom.set_seed(seed)
if tensorboard:
attack_logdir = join(run_logdir, attack)
attack_writer = tensorflow.summary.create_file_writer(
attack_logdir)
with attack_writer.as_default():
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
else:
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
training_steps_ls[attack] = training_steps
eve_scores_ls[attack] = sync_scores
losses_ls[attack] = loss
avg_training_steps = tensorflow.math.reduce_mean(
list(training_steps_ls.values()))
avg_eve_score = tensorflow.math.reduce_mean(
list(eve_scores_ls.values()))
mean_loss = tensorflow.math.reduce_mean(list(losses_ls.values()))
reporter(
avg_training_steps=avg_training_steps.numpy(),
avg_eve_score=avg_eve_score.numpy(),
mean_loss=mean_loss.numpy(),
done=True,
)
if algorithm == 'hyperopt':
from hyperopt import hp as hyperopt
from hyperopt.pyll.base import scope
from ray.tune.suggest.hyperopt import HyperOptSearch
space = {
'update_rule': hyperopt.choice(
'update_rule',
update_rules,
),
'K': scope.int(hyperopt.quniform('K', *K_bounds.values(), q=1)),
'N': scope.int(hyperopt.quniform('N', *N_bounds.values(), q=1)),
'L': scope.int(hyperopt.quniform('L', *L_bounds.values(), q=1)),
}
algo = HyperOptSearch(
space,
metric='mean_loss',
mode='min',
points_to_evaluate=[
{
'update_rule': 0,
'K': 3,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 128
},
],
)
elif algorithm == 'bayesopt':
from ray.tune.suggest.bayesopt import BayesOptSearch
space = {
'update_rule': (0, len(update_rules)),
'K': tuple(K_bounds.values()),
'N': tuple(N_bounds.values()),
'L': tuple(L_bounds.values()),
}
algo = BayesOptSearch(
space,
metric="mean_loss",
mode="min",
# TODO: what is utility_kwargs for and why is it needed?
utility_kwargs={
"kind": "ucb",
"kappa": 2.5,
"xi": 0.0
})
elif algorithm == 'nevergrad':
from ray.tune.suggest.nevergrad import NevergradSearch
from nevergrad import optimizers
from nevergrad import p as ngp
algo = NevergradSearch(
optimizers.TwoPointsDE(
ngp.Instrumentation(
update_rule=ngp.Choice(update_rules),
K=ngp.Scalar(lower=K_bounds['min'],
upper=K_bounds['max']).set_integer_casting(),
N=ngp.Scalar(lower=N_bounds['min'],
upper=N_bounds['max']).set_integer_casting(),
L=ngp.Scalar(lower=L_bounds['min'],
upper=L_bounds['max']).set_integer_casting(),
)),
None, # since the optimizer is already instrumented with kwargs
metric="mean_loss",
mode="min")
elif algorithm == 'skopt':
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
optimizer = Optimizer([
update_rules,
tuple(K_bounds.values()),
tuple(N_bounds.values()),
tuple(L_bounds.values())
])
algo = SkOptSearch(
optimizer,
["update_rule", "K", "N", "L"],
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'dragonfly':
# TODO: doesn't work
from ray.tune.suggest.dragonfly import DragonflySearch
from dragonfly.exd.experiment_caller import EuclideanFunctionCaller
from dragonfly.opt.gp_bandit import EuclideanGPBandit
# from dragonfly.exd.experiment_caller import CPFunctionCaller
# from dragonfly.opt.gp_bandit import CPGPBandit
from dragonfly import load_config
domain_config = load_config({
"domain": [
{
"name": "update_rule",
"type": "discrete",
"dim": 1,
"items": update_rules
},
{
"name": "K",
"type": "int",
"min": K_bounds['min'],
"max": K_bounds['max'],
# "dim": 1
},
{
"name": "N",
"type": "int",
"min": N_bounds['min'],
"max": N_bounds['max'],
# "dim": 1
},
{
"name": "L",
"type": "int",
"min": L_bounds['min'],
"max": L_bounds['max'],
# "dim": 1
}
]
})
func_caller = EuclideanFunctionCaller(
None, domain_config.domain.list_of_domains[0])
optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
algo = DragonflySearch(
optimizer,
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'bohb':
from ConfigSpace import ConfigurationSpace
from ConfigSpace import hyperparameters as CSH
from ray.tune.suggest.bohb import TuneBOHB
config_space = ConfigurationSpace()
config_space.add_hyperparameter(
CSH.CategoricalHyperparameter("update_rule", choices=update_rules))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='K',
lower=K_bounds['min'],
upper=K_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='N',
lower=N_bounds['min'],
upper=N_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='L',
lower=L_bounds['min'],
upper=L_bounds['max']))
algo = TuneBOHB(config_space, metric="mean_loss", mode="min")
elif algorithm == 'zoopt':
from ray.tune.suggest.zoopt import ZOOptSearch
from zoopt import ValueType
space = {
"update_rule":
(ValueType.DISCRETE, range(0, len(update_rules)), False),
"K": (ValueType.DISCRETE,
range(K_bounds['min'], K_bounds['max'] + 1), True),
"N": (ValueType.DISCRETE,
range(N_bounds['min'], N_bounds['max'] + 1), True),
"L": (ValueType.DISCRETE,
range(L_bounds['min'], L_bounds['max'] + 1), True),
}
# TODO: change budget to a large value
algo = ZOOptSearch(budget=10,
dim_dict=space,
metric="mean_loss",
mode="min")
# TODO: use more appropriate arguments for schedulers:
# https://docs.ray.io/en/master/tune/api_docs/schedulers.html
if scheduler == 'fifo':
sched = None # Tune defaults to FIFO
elif scheduler == 'pbt':
from ray.tune.schedulers import PopulationBasedTraining
from random import randint
sched = PopulationBasedTraining(
metric="mean_loss",
mode="min",
hyperparam_mutations={
"update_rule": update_rules,
"K": lambda: randint(K_bounds['min'], K_bounds['max']),
"N": lambda: randint(N_bounds['min'], N_bounds['max']),
"L": lambda: randint(L_bounds['min'], L_bounds['max']),
})
elif scheduler == 'ahb' or scheduler == 'asha':
# https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#asha-tune-schedulers-ashascheduler
from ray.tune.schedulers import AsyncHyperBandScheduler
sched = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
elif scheduler == 'hb':
from ray.tune.schedulers import HyperBandScheduler
sched = HyperBandScheduler(metric="mean_loss", mode="min")
elif algorithm == 'bohb' or scheduler == 'bohb':
from ray.tune.schedulers import HyperBandForBOHB
sched = HyperBandForBOHB(metric="mean_loss", mode="min")
elif scheduler == 'msr':
from ray.tune.schedulers import MedianStoppingRule
sched = MedianStoppingRule(metric="mean_loss", mode="min")
init_ray(
address=getenv("ip_head"),
redis_password=getenv('redis_password'),
)
analysis = tune.run(
trainable,
name='mlencrypt_research',
config={
"monitor": True,
"env_config": {
"wandb": {
"project": "mlencrypt-research",
"sync_tensorboard": True,
},
},
},
# resources_per_trial={"cpu": 1, "gpu": 3},
local_dir='./ray_results',
export_formats=['csv'], # TODO: add other formats?
num_samples=num_samples,
loggers=[
tune.logger.JsonLogger, tune.logger.CSVLogger,
tune.logger.TBXLogger, WandbLogger
],
search_alg=algo,
scheduler=sched,
queue_trials=True,
)
try:
wandbsweep(analysis)
except wandbCommError:
# see https://docs.wandb.com/sweeps/ray-tune#feature-compatibility
pass
best_config = analysis.get_best_config(metric='mean_loss', mode='min')
print(f"Best config: {best_config}")
shutdown_ray()
def _test_xgboost(method='BlendSearch'):
try:
import ray
except ImportError:
return
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
search_space = {
# You can mix constants with search space objects.
"max_depth": tune.randint(1, 8) if method in [
"BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight": tune.choice([1, 2, 3]),
"subsample": tune.uniform(0.5, 1.0),
"eta": tune.loguniform(1e-4, 1e-1)
}
max_iter = 10
for num_samples in [256]:
time_budget_s = 60 #None
for n_cpu in [8]:
start_time = time.time()
ray.init(num_cpus=n_cpu, num_gpus=0)
if method == 'BlendSearch':
analysis = tune.run(
train_breast_cancer,
init_config={
"max_depth": 1,
"min_child_weight": 3,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
report_intermediate_result=True,
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir='logs/',
num_samples=num_samples*n_cpu,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"max_depth": 1,
"min_child_weight": 3,
}], cat_hp_cost={
"min_child_weight": [6, 3, 2],
})
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples*n_cpu)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(
max_t=max_iter,
grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space, local_dir='logs/',
num_samples=num_samples*n_cpu, time_budget_s=time_budget_s,
scheduler=scheduler, search_alg=algo)
ray.shutdown()
# # Load the best model checkpoint
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss","min","all")
accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def testConvertBOHB(self):
from ray.tune.suggest.bohb import TuneBOHB
import ConfigSpace
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
TuneBOHB.convert_search_space({"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = TuneBOHB.convert_search_space(config)
bohb_config = ConfigSpace.ConfigurationSpace()
bohb_config.add_hyperparameters([
ConfigSpace.CategoricalHyperparameter("a", [2, 3, 4]),
ConfigSpace.UniformIntegerHyperparameter("b/x",
lower=0,
upper=4,
q=2),
ConfigSpace.UniformFloatHyperparameter("b/z",
lower=1e-4,
upper=1e-2,
log=True)
])
converted_config.seed(1234)
bohb_config.seed(1234)
searcher1 = TuneBOHB(space=converted_config, metric="a", mode="max")
searcher2 = TuneBOHB(space=bohb_config, metric="a", mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = TuneBOHB(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["a"], [2, 3, 4])
self.assertEqual(trial.config["b"]["y"], 4)
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix ConfigSpace and Dict
}
searcher = TuneBOHB(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
# Optional: Pass the parameter space yourself
# config_space = CS.ConfigurationSpace()
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("width", lower=0, upper=20))
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
# config_space.add_hyperparameter(
# CS.CategoricalHyperparameter(
# "activation", choices=["relu", "tanh"]))
bohb_hyperband = HyperBandForBOHB(time_attr="training_iteration",
max_t=100,
reduction_factor=4)
bohb_search = TuneBOHB(
# space=config_space, # If you want to set the space manually
max_concurrent=4)
analysis = tune.run(MyTrainableClass,
name="bohb_test",
config=config,
scheduler=bohb_hyperband,
search_alg=bohb_search,
num_samples=10,
stop={"training_iteration": 100},
metric="episode_reward_mean",
mode="max")
print("Best hyperparameters found were: ", analysis.best_config)
# Optional: Pass the parameter space yourself
# config_space = CS.ConfigurationSpace()
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("width", lower=0, upper=20))
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
# config_space.add_hyperparameter(
# CS.CategoricalHyperparameter(
# "activation", choices=["relu", "tanh"]))
experiment_metrics = dict(metric="episode_reward_mean", mode="max")
bohb_hyperband = HyperBandForBOHB(time_attr="training_iteration",
max_t=100,
reduction_factor=4,
**experiment_metrics)
bohb_search = TuneBOHB(
# space=config_space, # If you want to set the space manually
max_concurrent=4,
**experiment_metrics)
tune.run(MyTrainableClass,
name="bohb_test",
config=config,
scheduler=bohb_hyperband,
search_alg=bohb_search,
num_samples=10,
stop={"training_iteration": 100})