def _get_search_algorithm(
self, search_algorithm, config_space, metric, mode, max_concurrent):
if search_algorithm == "BO":
algo = BayesOptSearch(
utility_kwargs={
"kind": "ucb",
"kappa": 2.5,
"xi": 0.0
})
algo = ConcurrencyLimiter(algo, max_concurrent=max_concurrent)
scheduler = AsyncHyperBandScheduler()
elif search_algorithm == "BOHB":
experiment_metrics = dict(metric=metric, mode=mode)
algo = TuneBOHB(
config_space, max_concurrent=max_concurrent, **experiment_metrics)
scheduler = HyperBandForBOHB(
time_attr="training_iteration",
reduction_factor=4)
elif search_algorithm == "PBT":
# Problem of PBT: It mutates the param value, so sometimes, it generates unacceptable values
algo = None
scheduler = PopulationBasedTraining(
time_attr='training_iteration',
perturbation_interval=2, # Every N time_attr units, "perturb" the parameters.
hyperparam_mutations=config_space)
elif search_algorithm == "GRID" or search_algorithm == "RANDOM":
algo = None
scheduler = None
else:
raise Exception(search_algorithm, "is not available yet")
return algo, scheduler
def main(args):
cfg = setup(args)
search_space = CS.ConfigurationSpace()
search_space.add_hyperparameters([
CS.UniformFloatHyperparameter(name="lr", lower=1e-6, upper=1e-3),
CS.UniformFloatHyperparameter(name="wd", lower=0, upper=1e-3),
CS.UniformFloatHyperparameter(name="wd_bias", lower=0, upper=1e-3),
CS.CategoricalHyperparameter(name="bsz",
choices=[64, 96, 128, 160, 224, 256]),
CS.CategoricalHyperparameter(name="num_inst",
choices=[2, 4, 8, 16, 32]),
CS.UniformIntegerHyperparameter(name="delay_iters", lower=20,
upper=60),
CS.UniformFloatHyperparameter(name="ce_scale", lower=0.1, upper=1.0),
CS.UniformIntegerHyperparameter(name="circle_scale",
lower=8,
upper=256),
CS.UniformFloatHyperparameter(name="circle_margin",
lower=0.1,
upper=0.5),
CS.CategoricalHyperparameter(name="autoaug_enabled",
choices=[True, False]),
CS.CategoricalHyperparameter(name="cj_enabled", choices=[True, False]),
])
exp_metrics = dict(metric="score", mode="max")
bohb_hyperband = HyperBandForBOHB(
time_attr="training_iteration",
max_t=7,
**exp_metrics,
)
bohb_search = TuneBOHB(search_space, max_concurrent=4, **exp_metrics)
reporter = CLIReporter(parameter_columns=["bsz", "num_inst", "lr"],
metric_columns=["r1", "map", "training_iteration"])
analysis = tune.run(partial(train_reid_tune, cfg),
resources_per_trial={
"cpu": 10,
"gpu": 1
},
search_alg=bohb_search,
num_samples=args.num_samples,
scheduler=bohb_hyperband,
progress_reporter=reporter,
local_dir=cfg.OUTPUT_DIR,
keep_checkpoints_num=4,
name="bohb")
best_trial = analysis.get_best_trial("map", "max", "last")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation mAP: {}, Rank-1: {}".format(
best_trial.last_result["map"], best_trial.last_result["r1"]))
def setup_tune_scheduler():
config_space = workload.create_ch()
experiment_metrics = workload.exp_metric()
bohb_hyperband = HyperBandForBOHB(time_attr="training_iteration",
max_t=243,
reduction_factor=3,
**experiment_metrics)
bohb_search = TuneBOHB(config_space, **experiment_metrics)
return dict(
scheduler=bohb_hyperband,
search_alg=bohb_search,
resources_per_trial=com.detect_baseline_resource(),
)
"height": tune.uniform(-100, 100),
"activation": tune.choice(["relu", "tanh"])
}
# Optional: Pass the parameter space yourself
# config_space = CS.ConfigurationSpace()
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("width", lower=0, upper=20))
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
# config_space.add_hyperparameter(
# CS.CategoricalHyperparameter(
# "activation", choices=["relu", "tanh"]))
bohb_hyperband = HyperBandForBOHB(time_attr="training_iteration",
max_t=100,
reduction_factor=4)
bohb_search = TuneBOHB(
# space=config_space, # If you want to set the space manually
max_concurrent=4)
analysis = tune.run(MyTrainableClass,
name="bohb_test",
config=config,
scheduler=bohb_hyperband,
search_alg=bohb_search,
num_samples=10,
stop={"training_iteration": 100},
metric="episode_reward_mean",
mode="max")
def main(args):
cfg = setup(args)
exp_metrics = dict(metric="score", mode="max")
if args.srch_algo == "hyperopt":
# Create a HyperOpt search space
search_space = {
# "lr": hp.loguniform("lr", np.log(1e-6), np.log(1e-3)),
# "delay_epochs": hp.randint("delay_epochs", 20, 60),
# "wd": hp.uniform("wd", 0, 1e-3),
# "wd_bias": hp.uniform("wd_bias", 0, 1e-3),
"bsz": hp.choice("bsz", [64, 96, 128, 160, 224, 256]),
"num_inst": hp.choice("num_inst", [2, 4, 8, 16, 32]),
# "ce_scale": hp.uniform("ce_scale", 0.1, 1.0),
# "circle_scale": hp.choice("circle_scale", [16, 32, 64, 128, 256]),
# "circle_margin": hp.uniform("circle_margin", 0, 1) * 0.4 + 0.1,
}
current_best_params = [{
"bsz": 0, # index of hp.choice list
"num_inst": 3,
}]
search_algo = HyperOptSearch(search_space,
points_to_evaluate=current_best_params,
**exp_metrics)
if args.pbt:
scheduler = PopulationBasedTraining(
time_attr="training_iteration",
**exp_metrics,
perturbation_interval=2,
hyperparam_mutations={
"bsz": [64, 96, 128, 160, 224, 256],
"num_inst": [2, 4, 8, 16, 32],
})
else:
scheduler = ASHAScheduler(grace_period=2,
reduction_factor=3,
max_t=7,
**exp_metrics)
elif args.srch_algo == "bohb":
search_space = CS.ConfigurationSpace()
search_space.add_hyperparameters([
# CS.UniformFloatHyperparameter(name="lr", lower=1e-6, upper=1e-3, log=True),
# CS.UniformIntegerHyperparameter(name="delay_epochs", lower=20, upper=60),
# CS.UniformFloatHyperparameter(name="ce_scale", lower=0.1, upper=1.0),
# CS.UniformIntegerHyperparameter(name="circle_scale", lower=8, upper=128),
# CS.UniformFloatHyperparameter(name="circle_margin", lower=0.1, upper=0.5),
# CS.UniformFloatHyperparameter(name="wd", lower=0, upper=1e-3),
# CS.UniformFloatHyperparameter(name="wd_bias", lower=0, upper=1e-3),
CS.CategoricalHyperparameter(name="bsz",
choices=[64, 96, 128, 160, 224, 256]),
CS.CategoricalHyperparameter(name="num_inst",
choices=[2, 4, 8, 16, 32]),
# CS.CategoricalHyperparameter(name="autoaug_enabled", choices=[True, False]),
# CS.CategoricalHyperparameter(name="cj_enabled", choices=[True, False]),
])
search_algo = TuneBOHB(search_space, max_concurrent=4, **exp_metrics)
scheduler = HyperBandForBOHB(
time_attr="training_iteration",
reduction_factor=3,
max_t=7,
**exp_metrics,
)
else:
raise ValueError(
"Search algorithm must be chosen from [hyperopt, bohb], but got {}"
.format(args.srch_algo))
reporter = CLIReporter(parameter_columns=["bsz", "num_inst"],
metric_columns=["r1", "map", "training_iteration"])
analysis = tune.run(partial(train_tuner, cfg=cfg),
resources_per_trial={
"cpu": 4,
"gpu": 1
},
search_alg=search_algo,
num_samples=args.num_trials,
scheduler=scheduler,
progress_reporter=reporter,
local_dir=cfg.OUTPUT_DIR,
keep_checkpoints_num=10,
name=args.srch_algo)
best_trial = analysis.get_best_trial("score", "max", "last")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation mAP: {}, Rank-1: {}".format(
best_trial.last_result["map"], best_trial.last_result["r1"]))
save_dict = dict(R1=best_trial.last_result["r1"].item(),
mAP=best_trial.last_result["map"].item())
save_dict.update(best_trial.config)
path = os.path.join(cfg.OUTPUT_DIR, "best_config.yaml")
with PathManager.open(path, "w") as f:
f.write(CfgNode(save_dict).dump())
logger.info("Best config saved to {}".format(os.path.abspath(path)))
}
# Optional: Pass the parameter space yourself
# import ConfigSpace as CS
# config_space = CS.ConfigurationSpace()
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("width", lower=0, upper=20))
# config_space.add_hyperparameter(
# CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
# config_space.add_hyperparameter(
# CS.CategoricalHyperparameter(
# "activation", choices=["relu", "tanh"]))
bohb_hyperband = HyperBandForBOHB(
time_attr="training_iteration",
max_t=100,
reduction_factor=4,
stop_last_trials=False,
)
bohb_search = TuneBOHB(
# space=config_space, # If you want to set the space manually
)
bohb_search = tune.search.ConcurrencyLimiter(bohb_search, max_concurrent=4)
analysis = tune.run(
MyTrainableClass,
name="bohb_test",
config=config,
scheduler=bohb_hyperband,
search_alg=bohb_search,
num_samples=10,
def get_scheduler(self):
return HyperBandForBOHB(max_t=10, metric="loss", mode="min")
def main(args):
def trainable(config):
print('begin a trial')
args.params = tools.AttrDict(yaml.safe_load(args.params.replace('#', ',')))
args.logdir = args.logdir and os.path.expanduser(args.logdir)
print('debug ', config["divergence_scale"], config["reward_loss_scale"])
with args.params.unlocked:
args.params.divergence_scale = config["divergence_scale"]
args.params.reward_loss_scale = config["reward_loss_scale"]
# args.params.main_learning_rate = config["main_learning_rate"]
args.params.test_steps = 50
# args.params.num_units = config['num_units']
args.params.test_traj = 5
training.utility.set_up_logging()
experiment = training.Experiment(
args.logdir,
process_fn=functools.partial(process, args=args),
num_runs=args.num_runs,
ping_every=args.ping_every,
resume_runs=args.resume_runs)
for run in experiment:
for test_score in run:
if test_score > 1.0:
tune.report(mean_score=test_score)
break
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
# search = {
# "divergence_scale": tune.quniform(1, 30, 1),
# "reward_loss_scale": tune.quniform(1, 50, 1),
# }
search = {
"divergence_scale": tune.grid_search([0.1, 1, 2, 3, 5, 10]),
"reward_loss_scale": tune.grid_search([1, 2, 5, 10, 20]),
}
config_space = CS.ConfigurationSpace(seed=1234)
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name="divergence_scale", lower=1, upper=30))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name="reward_loss_scale", lower=1, upper=50))
# config_space.add_hyperparameter(
# CSH.UniformFloatHyperparameter("main_learning_rate", lower=0.0001, upper=0.05, log=True))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter("main_learning_rate", lower=1, upper=500, log=True))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name="num_units", lower=150, upper=400, q=50))
bayesopt = BayesOptSearch(metric="mean_loss", mode="min")
bohb_hyperband = HyperBandForBOHB(metric="mean_score", mode="max", time_attr="training_iteration", max_t=30,
reduction_factor=3)
bohb_search = TuneBOHB(space=config_space, max_concurrent=1, metric="mean_score", mode="max")
bayesopt = BayesOptSearch(max_concurrent=3, metric="mean_score", mode="max")
asha = ASHAScheduler(metric="mean_score", mode="max", grace_period=6, reduction_factor=3)
analysis = tune.run(
trainable,
config=search,
num_samples=3,
scheduler=asha,
resources_per_trial={"cpu": 16, "gpu": 1},
stop={"training_iteration": 13},
# search_alg=bayesopt,
log_to_file=True
)
df = analysis.results_df
print("Best config: ", analysis.get_best_config(
metric="mean_score", mode="min"))
print(df)
for k, v in result.items() if k in self._csv_out.fieldnames})
self._file.flush()
from ray.tune.logger import *
from ray import tune
from ray.tune.schedulers.hb_bohb import HyperBandForBOHB
from ray.tune.suggest.bohb import *
for _ in range(1):
ray.shutdown()
ray.init()
start_time = time.time()
algo = TuneBOHB(metric="loss", mode="min")
bohb = HyperBandForBOHB(time_attr="training_iteration",
metric="loss",
mode="min",
max_t=46)
analysis = tune.run(
train_mnist_pb2,
scheduler=bohb,
reuse_actors=True,
search_alg=algo,
verbose=2,
checkpoint_at_end=True,
num_samples=36,
# export_formats=[ExportFormat.MODEL],
config={
"lr": tune.loguniform(1e-8, .36),
"drp": tune.uniform(.05, .15),
"weight_decay": tune.loguniform(1e-4, 1e-2),
# losses.OpticalLoss(),
# losses.BinHeights(),
# losses.AC(),
# losses.ACLower(),
# losses.Motication(),
# losses.GaussACDC(),
],
))
add(cat("max_validation_steps", [100]))
add(cat("batches_per_step", [10]))
metric = dict(metric="validation_loss", mode="min")
scheduling = HyperBandForBOHB(time_attr="training_iteration",
max_t=500,
reduction_factor=3,
**metric)
search = TuneBOHB(config_space, **metric)
if __name__ == "__main__":
#sys.exit(10)
# ray.init(local_mode=True) # use only one thread to make debugging easier
analysis = tune.run(
trainable.MultipackTrainable,
name="MPT_long_run",
scheduler=scheduling,
search_alg=search,
num_samples=1000,
stop={"training_iteration": 500},
max_failures=0, # for debugging purposes
resources_per_trial={
