def set_basic_conf(self):
space_config = [
{
"name": "width",
"type": "num",
"lb": 0,
"ub": 20
},
{
"name": "height",
"type": "num",
"lb": -100,
"ub": 100
},
]
space = HEBODesignSpace().parse(space_config)
def cost(param, reporter):
reporter(loss=(param["height"] - 14)**2 - abs(param["width"] - 3))
search_alg = HEBOSearch(
space=space, metric="loss", mode="min", random_state_seed=5)
return search_alg, cost
{
"width": 10,
"height": 0,
"activation": "relu" # Activation will be relu
},
{
"width": 15,
"height": -20,
"activation": "tanh" # Activation will be tanh
}
]
known_rewards = [-189, -1144]
algo = HEBOSearch(
# space = space, # If you want to set the space
points_to_evaluate=previously_run_params,
evaluated_rewards=known_rewards,
random_state_seed=123, # for reproducibility
)
algo = ConcurrencyLimiter(algo, max_concurrent=4)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(easy_objective,
metric="mean_loss",
mode="min",
name="hebo_exp_with_warmstart",
search_alg=algo,
scheduler=scheduler,
num_samples=10 if args.smoke_test else 50,
config={
"steps": 100,
def testConvertHEBO(self):
from ray.tune.suggest.hebo import HEBOSearch
from hebo.design_space.design_space import DesignSpace
import torch
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
HEBOSearch.convert_search_space({"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = HEBOSearch.convert_search_space(config)
hebo_space_config = [
{
"name": "a",
"type": "cat",
"categories": [2, 3, 4]
},
{
"name": "b/x",
"type": "int",
"lb": 0,
"ub": 5
},
{
"name": "b/z",
"type": "pow",
"lb": 1e-4,
"ub": 1e-2
},
]
hebo_space = DesignSpace().parse(hebo_space_config)
searcher1 = HEBOSearch(space=converted_config,
metric="a",
mode="max",
random_state_seed=123)
searcher2 = HEBOSearch(space=hebo_space,
metric="a",
mode="max",
random_state_seed=123)
np.random.seed(1234)
torch.manual_seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
torch.manual_seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = HEBOSearch(metric="a", mode="max", random_state_seed=123)
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["a"], [2, 3, 4])
self.assertEqual(trial.config["b"]["y"], 4)
def ray_tune(model_wrapper, job_config: ht.config, resume: bool = False):
"""Performs automatic hyper-parameters tuning with Ray"""
# initialize
tuner = job_config.tune.clone().tuner
log_dir = pathlib.Path(job_config.run.save_sub_dir) / "tmp_log"
log_dir.mkdir(parents=True, exist_ok=True)
# set up config
config = get_hypers_tune(job_config)
# set up scheduler
sched_class = getattr(schedulers, tuner.scheduler_class)
logger.info(f"Setting up scheduler: {tuner.scheduler_class}")
sched_config = tuner.scheduler.get_config_dict()
sched = sched_class(**sched_config)
# set up algorithm
algo_class = tuner.algo_class
logger.info(f"Setting up search algorithm: {tuner.algo_class}")
algo_config = tuner.algo.get_config_dict()
algo = None
if algo_class is None:
algo = None
elif algo_class == "AxSearch":
from ray.tune.suggest.ax import AxSearch
algo = AxSearch(**algo_config)
elif algo_class == "HyperOptSearch":
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch(**algo_config)
elif algo_class == "HEBOSearch":
from ray.tune.suggest.hebo import HEBOSearch
algo = HEBOSearch(**algo_config)
else:
logger.error(f"Unsupported search algorithm: {algo_class}")
logger.info(f"Using default value None for search algorithm")
# set stopper
if tuner.stopper_class is None:
stop = None
else:
logger.info(f"Setting up stopper: {tuner.stopper_class}")
stop_class = getattr(ray.tune.stopper, tuner.stopper_class)
stop_config = tuner.stopper.get_config_dict()
stop = stop_class(**stop_config)
# set up extra run configs
run_config = (tuner.run.get_config_dict()
) # important: convert Hepy_Config class to dict
if not "raise_on_failed_trial" in run_config:
run_config["raise_on_failed_trial"] = False
tune_func = getattr(hep_model, model_wrapper._tune_fun_name)
# start tuning jobs
if os.name == "posix":
logger.info(f"Ignoring tune.tmp.tmp_dir setting on Unix OS")
ray.init(**(tuner.init.get_config_dict()))
else:
ray.init(
_temp_dir=str(job_config.tune.tmp_dir),
**(tuner.init.get_config_dict()),
)
analysis = tune.run(
tune_func,
name="ray_tunes",
stop=stop,
search_alg=algo,
scheduler=sched,
config=config,
local_dir=job_config.run.save_sub_dir,
resume=resume,
**run_config,
)
print("#### Best hyperparameters found were:")
print(analysis.best_config)
print(yaml.dump(analysis.best_config))
return analysis
def get_run_params(tuning_config, param_config):
params = {}
if "benchmarking_params" in tuning_config:
from benchmarking_trainable import BenchmarkingTrainable
params["run_or_experiment"] = BenchmarkingTrainable
else:
from tuning_trainable import TuningTrainable
from tuning_callbacks import TuningCallbacks
params["run_or_experiment"] = TuningTrainable
params["callbacks"] = [TuningCallbacks(tuning_config, param_config)]
params["name"] = tuning_config["name"]
params["num_samples"] = tuning_config["samples"]
if 'tuning_timeout' in tuning_config:
params["time_budget_s"] = tuning_config['tuning_timeout']
params["verbose"] = tuning_config["logging_level"]
#params["stop"] = { "training_iteration": tuning_config["stages"] }
params["resources_per_trial"] = {
"cpu":
multiprocessing.cpu_count() / tuning_config["max_concurrent_trials"],
"gpu": 0
}
params["mode"] = "max"
params["metric"] = "score"
finalize_param_config(tuning_config, param_config)
params["config"] = param_config
if tuning_config["use_wandb"]:
from ray.tune.integration.wandb import WandbLoggerCallback
callbacks = params["callbacks"] = params.get("callbacks", [])
wandb_cfg_f = open(os.path.expanduser("~") + "/.netrc", "r")
# last line, last word, without \n at the end
wandb_api_key = wandb_cfg_f.readlines()[-1].split(' ')[-1][:-1]
callbacks.append(
WandbLoggerCallback(project="ralph-tuning" if "benchmarking_params"
not in tuning_config else "ralph-benchmarking",
api_key=wandb_api_key,
group=tuning_config["name"],
log_config=False))
os.environ["WANDB_START_METHOD"] = "thread"
# silent=true is very important, do not delete;
# otherwise, the whole tuning gets mysteriously stuck
os.environ["WANDB_SILENT"] = "true"
params["reuse_actors"] = True
points_to_evaluate = tuning_config.get("initial_param_configs", None)
max_concurrent = tuning_config["max_concurrent_trials"]
if tuning_config["optimizer"] == "RANDOM":
from ray.tune.suggest import BasicVariantGenerator
params["search_alg"] = BasicVariantGenerator(
points_to_evaluate=points_to_evaluate,
max_concurrent=max_concurrent)
elif tuning_config["optimizer"] == "SKOPT":
from ray.tune.suggest.skopt import SkOptSearch
params["search_alg"] = SkOptSearch(
metric="score",
mode="max",
points_to_evaluate=points_to_evaluate,
convert_to_python=True,
max_concurrent=max_concurrent)
elif tuning_config["optimizer"] == "HEBO":
from ray.tune.suggest.hebo import HEBOSearch
params["search_alg"] = HEBOSearch(
metric="score",
mode="max",
points_to_evaluate=points_to_evaluate,
max_concurrent=max_concurrent)
return params
