def testSkopt(self):
from ray.tune.suggest.skopt import SkOptSearch
searcher = SkOptSearch(space=self.config, metric=self.metric_name, mode="max")
self._save(searcher)
searcher = SkOptSearch(space=self.config, metric=self.metric_name, mode="max")
self._restore(searcher)
def testConvertSkOpt(self):
from ray.tune.suggest.skopt import SkOptSearch
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = SkOptSearch.convert_search_space(config)
skopt_config = {"a": [2, 3, 4], "b/x": (0, 5), "b/z": (1e-4, 1e-2)}
searcher1 = SkOptSearch(space=converted_config)
searcher2 = SkOptSearch(space=skopt_config)
np.random.seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = SkOptSearch(metric="a", mode="max")
analysis = tune.run(
_mock_objective, config=config, search_alg=searcher, num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["a"], [2, 3, 4])
self.assertEqual(trial.config["b"]["y"], 4)
def setup_tune_scheduler():
from ray.tune.suggest.skopt import SkOptSearch
from ray.tune.suggest.suggestion import ConcurrencyLimiter
from skopt import Optimizer
exp_metrics = workload.exp_metric()
search_space, dim_names = workload.create_skopt_space()
algo = ConcurrencyLimiter(
SkOptSearch(
Optimizer(search_space),
dim_names,
**exp_metrics,
),
81,
)
scheduler = FluidBandScheduler(
max_res=81,
reduction_factor=3,
**exp_metrics,
)
return dict(
search_alg=algo,
scheduler=scheduler,
trial_executor=MyRayTrialExecutor(),
resources_per_trial=com.detect_baseline_resource(),
)
def run_batch(s):
f = open(
"/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_args",
"rb")
args = pickle.load(f)
f = open(
"/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_spaces",
"rb")
hyperspaces = pickle.load(f)
for i in s:
current_space = hyperspaces[i]
optimizer = Optimizer(current_space)
if args.model == "segmentation_cityscapes" or args.model == "segmentation_gis":
search_algo = SkOptSearch(
optimizer,
['learning_rate', 'epochs', 'batch_size', 'adam_epsilon'],
metric='average_res',
mode='max')
else:
search_algo = SkOptSearch(optimizer, [
'learning_rate', 'dropout', 'epochs', 'batch_size',
'adam_epsilon'
],
metric='average_res',
mode='max')
analysis = tune.run(
multi_train,
search_alg=search_algo,
num_samples=int(args.trials),
resources_per_trial={
'cpu': 25,
'gpu': 1
},
local_dir="/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/ray_results"
)
df = analysis.results_df
df_name = "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/" + args.out + "/"
df_name += "space_"
df_name += str(i)
df_name += ".csv"
df.to_csv(df_name)
print("Finished space " + args.space)
print(
"Finished all spaces. Files writtten to /lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/"
+ args.out)
def testConvergenceSkOpt(self):
from ray.tune.suggest.skopt import SkOptSearch
np.random.seed(0)
searcher = SkOptSearch()
analysis = self._testConvergence(searcher)
assert len(analysis.trials) < 100
assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-3)
def testSkopt(self):
from ray.tune.suggest.skopt import SkOptSearch
np.random.seed(1234) # At least one nan, inf, -inf and float
out = tune.run(_invalid_objective,
search_alg=SkOptSearch(),
config=self.config,
mode="max",
num_samples=8,
reuse_actors=False)
best_trial = out.best_trial
self.assertLessEqual(best_trial.config["report"], 2.0)
def set_basic_conf(self):
optimizer = skopt.Optimizer([(0, 20), (-100, 100)])
previously_run_params = [[10, 0], [15, -20]]
known_rewards = [-189, -1144]
def cost(space, reporter):
reporter(loss=(space["height"]**2 + space["width"]**2))
search_alg = SkOptSearch(optimizer, ["width", "height"],
metric="loss",
mode="min",
points_to_evaluate=previously_run_params,
evaluated_rewards=known_rewards)
return search_alg, cost
def testSkOpt(self):
from ray.tune.suggest.skopt import SkOptSearch
searcher = SkOptSearch(
space=self.space,
metric="metric",
mode="max",
)
point = {
self.param_name: self.valid_value,
}
get_len_X = lambda s: len(s._skopt_opt.Xi) # noqa E731
get_len_y = lambda s: len(s._skopt_opt.yi) # noqa E731
self.run_add_evaluated_point(point, searcher, get_len_X, get_len_y)
self.run_add_evaluated_trials(searcher, get_len_X, get_len_y)
def fit(self,
metric="log_test_fold_pvalue",
num_samples=10,
iterations=1,
max_concurrent=4,
distribute_deepprog=False,
timesteps_total=100):
"""
"""
self._distribute_deepprog = distribute_deepprog
config = {
"num_samples": num_samples,
"config": {
"iterations": iterations,
},
"stop": {
"timesteps_total": timesteps_total
},
}
metric_authorized = {
"log_test_fold_pvalue": "max",
"test_fold_cindex": "max",
"cluster_consistency": "max",
"log_full_pvalue": "max",
"sum_log_pval": "max",
"log_test_pval": "max",
"test_cindex": "max",
"mix_score": "max",
"test_consisentcy": "max",
}
try:
assert metric in metric_authorized
except Exception:
raise (Exception('{0} should be in {1}'.format(
metric, metric_authorized)))
optimizer_header = self.args_to_optimize.keys()
optimizer_value = [
self.args_to_optimize[key] for key in optimizer_header
]
# optimizer_value += [tuple(norm.items()) for norm in self.normalization]
optimizer = Optimizer(optimizer_value)
algo = SkOptSearch(
optimizer,
list(optimizer_header),
max_concurrent=max_concurrent,
metric=metric,
mode=metric_authorized[metric],
)
scheduler = AsyncHyperBandScheduler(metric=metric,
mode=metric_authorized[metric])
self.results = run(self._objective_only_training,
name=self.project_name,
search_alg=algo,
scheduler=scheduler,
**config)
index = ['config/' + key for key in self.args_to_optimize]
index = ['trial_name'] + index + \
["test_pval_{0}".format(key)
for key in self.test_datasets] + [metric, "full_pvalue"]
df = self.results.dataframe()[index]
print('#### best results obtained with:\n{0}'.format(
tabulate(df, headers='keys', tablefmt='psql')))
fname = '{0}/{1}_hyperparameter_scores_summary.tsv'.format(
self.path_results, self.project_name)
df.to_csv(fname, sep="\t")
print('File :{0} written'.format(fname))
def _test_roberta(method='BlendSearch'):
max_num_epoch = 100
num_samples = -1
time_budget_s = 3600
search_space = {
# You can mix constants with search space objects.
"num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
"learning_rate": flaml.tune.loguniform(1e-5, 3e-5),
"weight_decay": flaml.tune.uniform(0, 0.3),
"per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]),
"seed": flaml.tune.choice([12, 22, 33, 42]),
}
start_time = time.time()
ray.init(num_cpus=4, num_gpus=4)
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import tuneBOHB
algo = tuneBOHB(max_concurrent=4)
scheduler = HyperBandForBOHB(max_t=max_num_epoch)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'BlendSearch' == method:
from flaml import BlendSearch
algo = BlendSearch(
points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch(max_concurrent=3)
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1)
scheduler = None
analysis = ray.tune.run(train_roberta,
metric=HP_METRIC,
mode=MODE,
resources_per_trial={
"gpu": 4,
"cpu": 4
},
config=search_space,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
keep_checkpoints_num=1,
checkpoint_score_attr=HP_METRIC,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
metric = best_trial.metric_analysis[HP_METRIC][MODE]
logger.info(f"method={method}")
logger.info(f"n_trials={len(analysis.trials)}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _test_xgboost(method='BlendSearch'):
try:
import ray
except ImportError:
return
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
search_space = {
# You can mix constants with search space objects.
"max_depth": tune.randint(1, 8) if method in [
"BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight": tune.choice([1, 2, 3]),
"subsample": tune.uniform(0.5, 1.0),
"eta": tune.loguniform(1e-4, 1e-1)
}
max_iter = 10
for num_samples in [256]:
time_budget_s = 60 #None
for n_cpu in [8]:
start_time = time.time()
ray.init(num_cpus=n_cpu, num_gpus=0)
if method == 'BlendSearch':
analysis = tune.run(
train_breast_cancer,
init_config={
"max_depth": 1,
"min_child_weight": 3,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
report_intermediate_result=True,
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir='logs/',
num_samples=num_samples*n_cpu,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"max_depth": 1,
"min_child_weight": 3,
}], cat_hp_cost={
"min_child_weight": [6, 3, 2],
})
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples*n_cpu)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(
max_t=max_iter,
grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space, local_dir='logs/',
num_samples=num_samples*n_cpu, time_budget_s=time_budget_s,
scheduler=scheduler, search_alg=algo)
ray.shutdown()
# # Load the best model checkpoint
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss","min","all")
accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _tune_run(self, config, resources_per_trial):
"""Wrapper to call ``tune.run``. Multiple estimators are generated when
early stopping is possible, whereas a single estimator is
generated when early stopping is not possible.
Args:
config (dict): Configurations such as hyperparameters to run
``tune.run`` on.
resources_per_trial (dict): Resources to use per trial within Ray.
Accepted keys are `cpu`, `gpu` and custom resources, and values
are integers specifying the number of each resource to use.
Returns:
analysis (`ExperimentAnalysis`): Object returned by
`tune.run`.
"""
if self.early_stopping is not None:
config["estimator"] = [
clone(self.estimator) for _ in range(self.n_splits)
]
else:
config["estimator"] = self.estimator
if self.search_optimization == "random":
if isinstance(self.param_distributions, list):
analysis = tune.run(
_Trainable,
scheduler=self.early_stopping,
search_alg=RandomListSearcher(self.param_distributions),
reuse_actors=True,
verbose=self.verbose,
stop={"training_iteration": self.max_iters},
num_samples=self.num_samples,
config=config,
fail_fast=True,
checkpoint_at_end=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
else:
analysis = tune.run(
_Trainable,
scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop={"training_iteration": self.max_iters},
num_samples=self.num_samples,
config=config,
fail_fast=True,
checkpoint_at_end=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
else:
hyperparameter_names, spaces = self._get_skopt_params()
search_algo = SkOptSearch(
Optimizer(spaces),
hyperparameter_names,
metric="average_test_score")
analysis = tune.run(
_Trainable,
search_alg=search_algo,
scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop={"training_iteration": self.max_iters},
num_samples=self.num_samples,
config=config,
fail_fast=True,
checkpoint_at_end=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
return analysis
config = {
"num_samples": 10 if args.smoke_test else 50,
"config": {
"iterations": 100,
},
"stop": {
"timesteps_total": 100
},
}
optimizer = Optimizer([(0, 20), (-100, 100)])
previously_run_params = [[10, 0], [15, -20]]
known_rewards = [-189, -1144]
algo = SkOptSearch(optimizer, ["width", "height"],
max_concurrent=4,
metric="mean_loss",
mode="min",
points_to_evaluate=previously_run_params,
evaluated_rewards=known_rewards)
scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
run(easy_objective,
name="skopt_exp_with_warmstart",
search_alg=algo,
scheduler=scheduler,
**config)
# Now run the experiment without known rewards
algo = SkOptSearch(optimizer, ["width", "height"],
max_concurrent=4,
metric="mean_loss",
mode="min",
def _tune_run(self, config, resources_per_trial):
"""Wrapper to call ``tune.run``. Multiple estimators are generated when
early stopping is possible, whereas a single estimator is
generated when early stopping is not possible.
Args:
config (dict): Configurations such as hyperparameters to run
``tune.run`` on.
resources_per_trial (dict): Resources to use per trial within Ray.
Accepted keys are `cpu`, `gpu` and custom resources, and values
are integers specifying the number of each resource to use.
Returns:
analysis (`ExperimentAnalysis`): Object returned by
`tune.run`.
"""
if self.seed is not None:
random.seed(self.seed)
np.random.seed(self.seed)
trainable = _Trainable
if self.pipeline_auto_early_stop and check_is_pipeline(
self.estimator) and self.early_stopping:
trainable = _PipelineTrainable
max_iter = self.max_iters
if self.early_stopping is not None:
config["estimator_list"] = [
clone(self.estimator) for _ in range(self.n_splits)
]
if hasattr(self.early_stopping, "_max_t_attr"):
# we want to delegate stopping to schedulers which
# support it, but we want it to stop eventually, just in case
# the solution is to make the stop condition very big
max_iter = self.max_iters * 10
else:
config["estimator_list"] = [self.estimator]
stopper = MaximumIterationStopper(max_iter=max_iter)
if self.stopper:
stopper = CombinedStopper(stopper, self.stopper)
run_args = dict(scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stopper,
num_samples=self.n_trials,
config=config,
fail_fast="raise",
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir),
loggers=self.loggers,
time_budget_s=self.time_budget_s)
if self.search_optimization == "random":
if isinstance(self.param_distributions, list):
search_algo = RandomListSearcher(self.param_distributions)
else:
search_algo = BasicVariantGenerator()
run_args["search_alg"] = search_algo
else:
search_space = None
override_search_space = True
if self._is_param_distributions_all_tune_domains():
run_args["config"].update(self.param_distributions)
override_search_space = False
search_kwargs = self.search_kwargs.copy()
search_kwargs.update(metric=self._metric_name, mode="max")
if self.search_optimization == "bayesian":
from ray.tune.suggest.skopt import SkOptSearch
if override_search_space:
search_space = self.param_distributions
search_algo = SkOptSearch(space=search_space, **search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "bohb":
from ray.tune.suggest.bohb import TuneBOHB
if override_search_space:
search_space = self._get_bohb_config_space()
if self.seed:
warnings.warn("'seed' is not implemented for BOHB.")
search_algo = TuneBOHB(space=search_space, **search_kwargs)
# search_algo = TuneBOHB(
# space=search_space, seed=self.seed, **search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "optuna":
from ray.tune.suggest.optuna import OptunaSearch
from optuna.samplers import TPESampler
sampler = TPESampler(seed=self.seed)
if override_search_space:
search_space = self._get_optuna_params()
search_algo = OptunaSearch(space=search_space,
sampler=sampler,
**search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "hyperopt":
from ray.tune.suggest.hyperopt import HyperOptSearch
if override_search_space:
search_space = self._get_hyperopt_params()
search_algo = HyperOptSearch(space=search_space,
random_state_seed=self.seed,
**search_kwargs)
run_args["search_alg"] = search_algo
else:
# This should not happen as we validate the input before
# this method. Still, just to be sure, raise an error here.
raise ValueError(
f"Invalid search optimizer: {self.search_optimization}")
if isinstance(self.n_jobs, int) and self.n_jobs > 0 \
and not self.search_optimization == "random":
search_algo = ConcurrencyLimiter(search_algo,
max_concurrent=self.n_jobs)
run_args["search_alg"] = search_algo
with warnings.catch_warnings():
warnings.filterwarnings("ignore",
message="fail_fast='raise' "
"detected.")
analysis = tune.run(trainable, **run_args)
return analysis
def main(args):
# 1. load config
print('Importing architecture from %s' % args.arch_module)
arch_mod = import_module(args.arch_module)
prob_mods = []
for prob_module_path in args.prob_modules:
print('Importing problem from %s' % prob_module_path)
this_prob_mod = import_module(prob_module_path)
prob_mods.append(this_prob_mod)
# 2. spool up Ray
new_cluster = args.ray_connect is None
ray_kwargs = {}
if not new_cluster:
ray_kwargs["redis_address"] = args.ray_connect
assert args.ray_ncpus is None, \
"can't provide --ray-ncpus and --ray-connect"
else:
if args.ray_ncpus is not None:
assert args.job_ncpus is None \
or args.job_ncpus <= args.ray_ncpus, \
"must have --job-ncpus <= --ray-ncpus if both given"
ray_kwargs["num_cpus"] = args.ray_ncpus
ray.init(**ray_kwargs)
max_par_trials = args.max_par_trials
if max_par_trials is None:
# leave some room for hyperthread-caused over-counting of CPUs (a /2
# factor), and for running eval trials in parallel
max_par_trials = max(1, multiprocessing.cpu_count() // 5)
sk_space = OrderedDict()
# originally I had this split between 2/3, but I think 3 is a bit too slow
# on some problems, so I want to stick to 2 (even though exbw really seems
# to benefit from 3)
sk_space['num_layers'] = [2]
sk_space['hidden_size'] = (12, 20)
# empty list; no steps down, just a single fixed learning rate
sk_space['learning_rate_steps'] = [()]
sk_space['supervised_learning_rate'] = (1e-4, 1e-2, 'log-uniform')
# these ranges are similar to my original config, which seemed to work okay
sk_space['supervised_batch_size'] = (48, 128)
sk_space['opt_batch_per_epoch'] = (300, 1200) # (150, 1500)
# we use categorical vars to add "switched off entirely" as options (as
# opposed to just "turned down very low"); I suspect switching off entirely
# is good for some of those things
sk_space['dropout'] = [0, 0.1, 0.25]
sk_space['l1_reg'] = [0.0] # (1e-10, 1e-2, 'log-uniform')
sk_space['l2_reg'] = (1e-5, 1e-2, 'log-uniform')
sk_space['target_rollouts_per_epoch'] = (30, 150)
if arch_mod.TEACHER_PLANNER == 'ssipp':
# only relevant for SSiPP
# (originally I had both h-add and lm-cut as options, but lm-cut didn't
# seem to help much, so I'm leaving it out)
sk_space['ssipp_teacher_heuristic'] = ['h-add']
# using random forest b/c we have lots of discrete params, & a few
# categorical
sk_optimiser = Optimizer(list(sk_space.values()), base_estimator='RF')
algo = SkOptSearch(
sk_optimiser,
sk_space.keys(),
max_concurrent=max_par_trials,
metric='coverage',
mode='max')
perform_trial = make_perform_trial(arch_mod, prob_mods)
tune.run(
perform_trial,
search_alg=algo,
local_dir=args.work_dir,
resources_per_trial={"cpu": 0},
num_samples=1000)
def build_search_alg(search_alg, param_ranges: dict):
"""
Initialize a search algorithm that is selected using 'search_alg'
Parameters
----------
search_alg : str; Selecting the search algorithm. Possible values
[BayesOpt, SkOpt]
param_ranges : dictionary of parameter ranges over which the search
should be performed
Returns
-------
alg : Object of the RayTune search algorithm selected
"""
alg = None
if search_alg == "BayesOpt":
from ray.tune.suggest.bayesopt import BayesOptSearch
alg = BayesOptSearch(
param_ranges,
max_concurrent=max_concurrent,
metric="test_accuracy",
mode="max",
utility_kwargs={
"kind": "ucb",
"kappa": 2.5,
"xi": 0.0
},
)
elif search_alg == "SkOpt":
from skopt import Optimizer
from skopt.space import Real, Integer
from ray.tune.suggest.skopt import SkOptSearch
opt_params = [
Integer(param_ranges["n_estimators"][0],
param_ranges["n_estimators"][1]),
Integer(param_ranges["max_depth"][0],
param_ranges["max_depth"][1]),
Real(
param_ranges["max_features"][0],
param_ranges["max_features"][1],
prior="log-uniform",
),
]
optimizer = Optimizer(opt_params)
alg = SkOptSearch(
optimizer,
list(param_ranges.keys()),
max_concurrent=max_concurrent,
metric="test_accuracy",
mode="max",
)
else:
print("Unknown Option. Select BayesOpt or SkOpt")
return alg
def get_raytune_search_alg(raytune_cfg, seeds=False):
if (raytune_cfg["sched"] == "pbt") or (raytune_cfg["sched"] == "pb2"):
if raytune_cfg["search_alg"] is not None:
print(
"INFO: Using schedule '{}' is not compatible with Ray Tune search algorithms."
.format(raytune_cfg["sched"]))
print(
"INFO: Uing the Ray Tune {} scheduler without search algorithm"
.format(raytune_cfg["sched"]))
return None
if (raytune_cfg["sched"] == "bohb") or (raytune_cfg["sched"] == "BOHB"):
print(
"INFO: Using TuneBOHB search algorithm since it is required for BOHB shedule"
)
if seeds:
seed = 1234
else:
seed = None
return TuneBOHB(metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
seed=seed)
# requires pip install bayesian-optimization
if raytune_cfg["search_alg"] == "bayes":
print("INFO: Using BayesOptSearch")
return BayesOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
random_search_steps=raytune_cfg["bayes"]["n_random_steps"],
)
# requires pip install hyperopt
if raytune_cfg["search_alg"] == "hyperopt":
print("INFO: Using HyperOptSearch")
return HyperOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
n_initial_points=raytune_cfg["hyperopt"]["n_random_steps"],
# points_to_evaluate=,
)
if raytune_cfg["search_alg"] == "scikit":
print("INFO: Using bayesian optimization from scikit-learn")
return SkOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
convert_to_python=True,
)
if raytune_cfg["search_alg"] == "nevergrad":
print("INFO: Using bayesian optimization from nevergrad")
return NevergradSearch(
optimizer=ng.optimizers.BayesOptim(
pca=False,
init_budget=raytune_cfg["nevergrad"]["n_random_steps"]),
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
)
# HEBO is not yet supported
# if (raytune_cfg["search_alg"] == "hebo") or (raytune_cfg["search_alg"] == "HEBO"):
# print("Using HEBOSearch")
# return HEBOSearch(
# metric=raytune_cfg["default_metric"],
# mode=raytune_cfg["default_mode"],
# # max_concurrent=8,
# )
else:
print("INFO: Not using any Ray Tune search algorithm")
return None
ncalls = 20
if TT:
parameter_names = ['lr']
space = [Real(10**-7, 10**-3, "log-uniform", name='lr')]
else:
parameter_names = ['lr']
space = [Real(10**-7, 10**-3, "log-uniform", name='lr')]
ray.init(num_cpus=nCPU, num_gpus=nGPU)
optimizer = Optimizer(dimensions=space,
random_state=1,
base_estimator='gp')
algo = SkOptSearch(optimizer,
parameter_names=parameter_names,
max_concurrent=4,
metric="result",
mode="max")
scheduler = FIFOScheduler()
tune.register_trainable("train_func", train)
import time, random
time.sleep(random.uniform(0.0, 10.0))
tune.run_experiments(
{
'my_experiment': {
'run': 'train_func',
'resources_per_trial': {
"cpu": int(nCPU * load // nGPU),
"gpu": load
},
def compile(
self,
input_df,
model_create_func,
search_space,
recipe,
feature_transformers=None,
# model=None,
future_seq_len=1,
validation_df=None,
mc=False,
metric="mse",
metric_mode="min"):
"""
Do necessary preparations for the engine
:param input_df:
:param search_space:
:param num_samples:
:param stop:
:param search_algorithm:
:param search_algorithm_params:
:param fixed_params:
:param feature_transformers:
:param model:
:param validation_df:
:param metric:
:return:
"""
# prepare parameters for search engine
runtime_params = recipe.runtime_params()
num_samples = runtime_params['num_samples']
stop = dict(runtime_params)
search_algorithm_params = recipe.search_algorithm_params()
search_algorithm = recipe.search_algorithm()
fixed_params = recipe.fixed_params()
schedule_algorithm = recipe.scheduler_algorithm()
del stop['num_samples']
self.search_space = self._prepare_tune_config(search_space)
self.stop_criteria = stop
self.num_samples = num_samples
if schedule_algorithm == 'AsyncHyperBand':
from ray.tune.schedulers import AsyncHyperBandScheduler
self.sched = AsyncHyperBandScheduler(
time_attr="training_iteration",
metric="reward_metric",
mode="max",
max_t=50,
grace_period=1,
reduction_factor=3,
brackets=3,
)
else:
from ray.tune.schedulers import FIFOScheduler
self.sched = FIFOScheduler()
if search_algorithm == 'BayesOpt':
self.search_algorithm = BayesOptSearch(
self.search_space,
metric="reward_metric",
mode="max",
utility_kwargs=search_algorithm_params["utility_kwargs"])
elif search_algorithm == 'SkOpt':
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
opt_params = recipe.opt_params()
optimizer = Optimizer(opt_params)
self.search_algorithm = SkOptSearch(
optimizer,
list(self.search_space.keys()),
metric="reward_metric",
mode="max",
)
else:
self.search_algorithm = None
self.fixed_params = fixed_params
self.train_func = self._prepare_train_func(
input_df=input_df,
model_create_func=model_create_func,
feature_transformers=feature_transformers,
future_seq_len=future_seq_len,
validation_df=validation_df,
metric=metric,
metric_mode=metric_mode,
mc=mc,
remote_dir=self.remote_dir)
def _test_xgboost(method="BlendSearch"):
try:
import ray
except ImportError:
return
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
search_space = {
"max_depth":
tune.randint(1, 9)
if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight":
tune.choice([1, 2, 3]),
"subsample":
tune.uniform(0.5, 1.0),
"eta":
tune.loguniform(1e-4, 1e-1),
}
max_iter = 10
for num_samples in [128]:
time_budget_s = 60
for n_cpu in [2]:
start_time = time.time()
# ray.init(address='auto')
if method == "BlendSearch":
analysis = tune.run(
train_breast_cancer,
config=search_space,
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
scheduler="asha",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif "CFO" == method:
from flaml import CFO
algo = CFO(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "CFOCat" == method:
from flaml.searcher.cfo_cat import CFOCat
algo = CFOCat(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "Dragonfly" == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif "SkOpt" == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif "ZOOpt" == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples * n_cpu)
elif "Ax" == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif "HyperOpt" == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_iter, grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
# # Load the best model checkpoint
# import os
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss", "min", "all")
accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def hparams(algorithm, scheduler, num_samples, tensorboard, bare):
from glob import glob
import tensorflow.summary
from tensorflow import random as tfrandom, int64 as tfint64
from ray import init as init_ray, shutdown as shutdown_ray
from ray import tune
from wandb.ray import WandbLogger
from wandb import sweep as wandbsweep
from wandb.apis import CommError as wandbCommError
# less summaries are logged if MLENCRYPT_TB is TRUE (for efficiency)
# TODO: use tf.summary.record_if?
environ["MLENCRYPT_TB"] = str(tensorboard).upper()
environ["MLENCRYPT_BARE"] = str(bare).upper()
if getenv('MLENCRYPT_TB', 'FALSE') == 'TRUE' and \
getenv('MLENCRYPT_BARE', 'FALSE') == 'TRUE':
raise ValueError('TensorBoard logging cannot be enabled in bare mode.')
logdir = f'logs/hparams/{datetime.now()}'
# "These results show that K = 3 is the optimal choice for the
# cryptographic application of neural synchronization. K = 1 and K = 2 are
# too insecure in regard to the geometric attack. And for K > 3 the effort
# of A and B grows exponentially with increasing L, while the simple attack
# is quite successful in the limit K -> infinity. Consequently, one should
# only use Tree Parity Machines with three hidden units for the neural
# key-exchange protocol." (Ruttor, 2006)
# https://arxiv.org/pdf/0711.2411.pdf#page=59
update_rules = [
'random-same',
# 'random-different-A-B-E', 'random-different-A-B',
'hebbian',
'anti_hebbian',
'random_walk'
]
K_bounds = {'min': 4, 'max': 8}
N_bounds = {'min': 4, 'max': 8}
L_bounds = {'min': 4, 'max': 8}
# TODO: don't use *_bounds.values() since .values doesn't preserve order
def get_session_num(logdir):
current_runs = glob(join(logdir, "run-*"))
if current_runs:
last_run_path = current_runs[-1]
last_run_session_num = int(last_run_path.split('-')[-1])
return last_run_session_num + 1
else: # there are no runs yet, start at 0
return 0
def trainable(config, reporter):
"""
Args:
config (dict): Parameters provided from the search algorithm
or variant generation.
"""
if not isinstance(config['update_rule'], str):
update_rule = update_rules[int(config['update_rule'])]
else:
update_rule = config['update_rule']
K, N, L = int(config['K']), int(config['N']), int(config['L'])
run_name = f"run-{get_session_num(logdir)}"
run_logdir = join(logdir, run_name)
# for each attack, the TPMs should start with the same weights
initial_weights_tensors = get_initial_weights(K, N, L)
training_steps_ls = {}
eve_scores_ls = {}
losses_ls = {}
# for each attack, the TPMs should use the same inputs
seed = tfrandom.uniform([],
minval=0,
maxval=tfint64.max,
dtype=tfint64).numpy()
for attack in ['none', 'geometric']:
initial_weights = {
tpm: weights_tensor_to_variable(weights, tpm)
for tpm, weights in initial_weights_tensors.items()
}
tfrandom.set_seed(seed)
if tensorboard:
attack_logdir = join(run_logdir, attack)
attack_writer = tensorflow.summary.create_file_writer(
attack_logdir)
with attack_writer.as_default():
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
else:
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
training_steps_ls[attack] = training_steps
eve_scores_ls[attack] = sync_scores
losses_ls[attack] = loss
avg_training_steps = tensorflow.math.reduce_mean(
list(training_steps_ls.values()))
avg_eve_score = tensorflow.math.reduce_mean(
list(eve_scores_ls.values()))
mean_loss = tensorflow.math.reduce_mean(list(losses_ls.values()))
reporter(
avg_training_steps=avg_training_steps.numpy(),
avg_eve_score=avg_eve_score.numpy(),
mean_loss=mean_loss.numpy(),
done=True,
)
if algorithm == 'hyperopt':
from hyperopt import hp as hyperopt
from hyperopt.pyll.base import scope
from ray.tune.suggest.hyperopt import HyperOptSearch
space = {
'update_rule': hyperopt.choice(
'update_rule',
update_rules,
),
'K': scope.int(hyperopt.quniform('K', *K_bounds.values(), q=1)),
'N': scope.int(hyperopt.quniform('N', *N_bounds.values(), q=1)),
'L': scope.int(hyperopt.quniform('L', *L_bounds.values(), q=1)),
}
algo = HyperOptSearch(
space,
metric='mean_loss',
mode='min',
points_to_evaluate=[
{
'update_rule': 0,
'K': 3,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 128
},
],
)
elif algorithm == 'bayesopt':
from ray.tune.suggest.bayesopt import BayesOptSearch
space = {
'update_rule': (0, len(update_rules)),
'K': tuple(K_bounds.values()),
'N': tuple(N_bounds.values()),
'L': tuple(L_bounds.values()),
}
algo = BayesOptSearch(
space,
metric="mean_loss",
mode="min",
# TODO: what is utility_kwargs for and why is it needed?
utility_kwargs={
"kind": "ucb",
"kappa": 2.5,
"xi": 0.0
})
elif algorithm == 'nevergrad':
from ray.tune.suggest.nevergrad import NevergradSearch
from nevergrad import optimizers
from nevergrad import p as ngp
algo = NevergradSearch(
optimizers.TwoPointsDE(
ngp.Instrumentation(
update_rule=ngp.Choice(update_rules),
K=ngp.Scalar(lower=K_bounds['min'],
upper=K_bounds['max']).set_integer_casting(),
N=ngp.Scalar(lower=N_bounds['min'],
upper=N_bounds['max']).set_integer_casting(),
L=ngp.Scalar(lower=L_bounds['min'],
upper=L_bounds['max']).set_integer_casting(),
)),
None, # since the optimizer is already instrumented with kwargs
metric="mean_loss",
mode="min")
elif algorithm == 'skopt':
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
optimizer = Optimizer([
update_rules,
tuple(K_bounds.values()),
tuple(N_bounds.values()),
tuple(L_bounds.values())
])
algo = SkOptSearch(
optimizer,
["update_rule", "K", "N", "L"],
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'dragonfly':
# TODO: doesn't work
from ray.tune.suggest.dragonfly import DragonflySearch
from dragonfly.exd.experiment_caller import EuclideanFunctionCaller
from dragonfly.opt.gp_bandit import EuclideanGPBandit
# from dragonfly.exd.experiment_caller import CPFunctionCaller
# from dragonfly.opt.gp_bandit import CPGPBandit
from dragonfly import load_config
domain_config = load_config({
"domain": [
{
"name": "update_rule",
"type": "discrete",
"dim": 1,
"items": update_rules
},
{
"name": "K",
"type": "int",
"min": K_bounds['min'],
"max": K_bounds['max'],
# "dim": 1
},
{
"name": "N",
"type": "int",
"min": N_bounds['min'],
"max": N_bounds['max'],
# "dim": 1
},
{
"name": "L",
"type": "int",
"min": L_bounds['min'],
"max": L_bounds['max'],
# "dim": 1
}
]
})
func_caller = EuclideanFunctionCaller(
None, domain_config.domain.list_of_domains[0])
optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
algo = DragonflySearch(
optimizer,
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'bohb':
from ConfigSpace import ConfigurationSpace
from ConfigSpace import hyperparameters as CSH
from ray.tune.suggest.bohb import TuneBOHB
config_space = ConfigurationSpace()
config_space.add_hyperparameter(
CSH.CategoricalHyperparameter("update_rule", choices=update_rules))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='K',
lower=K_bounds['min'],
upper=K_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='N',
lower=N_bounds['min'],
upper=N_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='L',
lower=L_bounds['min'],
upper=L_bounds['max']))
algo = TuneBOHB(config_space, metric="mean_loss", mode="min")
elif algorithm == 'zoopt':
from ray.tune.suggest.zoopt import ZOOptSearch
from zoopt import ValueType
space = {
"update_rule":
(ValueType.DISCRETE, range(0, len(update_rules)), False),
"K": (ValueType.DISCRETE,
range(K_bounds['min'], K_bounds['max'] + 1), True),
"N": (ValueType.DISCRETE,
range(N_bounds['min'], N_bounds['max'] + 1), True),
"L": (ValueType.DISCRETE,
range(L_bounds['min'], L_bounds['max'] + 1), True),
}
# TODO: change budget to a large value
algo = ZOOptSearch(budget=10,
dim_dict=space,
metric="mean_loss",
mode="min")
# TODO: use more appropriate arguments for schedulers:
# https://docs.ray.io/en/master/tune/api_docs/schedulers.html
if scheduler == 'fifo':
sched = None # Tune defaults to FIFO
elif scheduler == 'pbt':
from ray.tune.schedulers import PopulationBasedTraining
from random import randint
sched = PopulationBasedTraining(
metric="mean_loss",
mode="min",
hyperparam_mutations={
"update_rule": update_rules,
"K": lambda: randint(K_bounds['min'], K_bounds['max']),
"N": lambda: randint(N_bounds['min'], N_bounds['max']),
"L": lambda: randint(L_bounds['min'], L_bounds['max']),
})
elif scheduler == 'ahb' or scheduler == 'asha':
# https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#asha-tune-schedulers-ashascheduler
from ray.tune.schedulers import AsyncHyperBandScheduler
sched = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
elif scheduler == 'hb':
from ray.tune.schedulers import HyperBandScheduler
sched = HyperBandScheduler(metric="mean_loss", mode="min")
elif algorithm == 'bohb' or scheduler == 'bohb':
from ray.tune.schedulers import HyperBandForBOHB
sched = HyperBandForBOHB(metric="mean_loss", mode="min")
elif scheduler == 'msr':
from ray.tune.schedulers import MedianStoppingRule
sched = MedianStoppingRule(metric="mean_loss", mode="min")
init_ray(
address=getenv("ip_head"),
redis_password=getenv('redis_password'),
)
analysis = tune.run(
trainable,
name='mlencrypt_research',
config={
"monitor": True,
"env_config": {
"wandb": {
"project": "mlencrypt-research",
"sync_tensorboard": True,
},
},
},
# resources_per_trial={"cpu": 1, "gpu": 3},
local_dir='./ray_results',
export_formats=['csv'], # TODO: add other formats?
num_samples=num_samples,
loggers=[
tune.logger.JsonLogger, tune.logger.CSVLogger,
tune.logger.TBXLogger, WandbLogger
],
search_alg=algo,
scheduler=sched,
queue_trials=True,
)
try:
wandbsweep(analysis)
except wandbCommError:
# see https://docs.wandb.com/sweeps/ray-tune#feature-compatibility
pass
best_config = analysis.get_best_config(metric='mean_loss', mode='min')
print(f"Best config: {best_config}")
shutdown_ray()
def create_optimization_algorithm(self, hyperparameter_space):
""" Create optimization algorithm for Ray Tune.
Currently, only tree parzen estimators, random search,
gaussian processes and genetic algorithm based optimization using Ray
Tune is supported.
Parameters:
space (dict, list or ray.tune.automl.search_space.SearchSpace):
space of hyperparameters following the syntax required by
the optimization algorithm.
Returns:
Optimization algorithm for Ray Tune.
Raises:
NotImplementedError: if self.optimization_type is other than
``tree_parzen_estimators``, ``random_search``,
``gaussian_processes`` or ``genetic_algorithm``.
"""
if self.optimization_type == 'tree_parzen_estimators':
algorithm = HyperOptSearch(hyperparameter_space,
max_concurrent=self.num_parallel_trials,
metric='loss',
mode=self.mode,
n_initial_points=self.n_initial_points,
random_state_seed=self.random_state,
gamma=self.tpe_config.get(
'gamma', 0.25))
elif self.optimization_type == 'random_search':
algorithm = HyperOptSearch(hyperparameter_space,
max_concurrent=self.num_parallel_trials,
metric='loss',
mode=self.mode,
random_state_seed=self.random_state)
algorithm.algo = hpo.rand.suggest
elif self.optimization_type == 'gaussian_processes':
if not self.reload_trials:
# the gaussian processes based optimization needs an
# extra parameter self.gp_opt
self.gp_opt = skopt.Optimizer(
hyperparameter_space,
n_initial_points=self.n_initial_points,
base_estimator=self.gp_config.get('base_estimator', 'GP'),
acq_func=self.gp_config.get('acq_function', 'gp_hedge'),
acq_optimizer=self.gp_config.get('acq_optimizer', 'auto'),
random_state=self.random_state,
acq_func_kwargs={
'xi': self.gp_config.get('xi', 0.01),
'kappa': self.gp_config.get('kappa', 1.96)
})
hyperparams_names = [key for key in self.hyperparams_to_optimize]
algorithm = SkOptSearch(self.gp_opt,
hyperparams_names,
max_concurrent=self.num_parallel_trials,
metric='loss',
mode=self.mode)
elif self.optimization_type == 'genetic_algorithm':
algorithm = GeneticSearch(
hyperparameter_space,
reward_attr='loss',
max_generation=self.ga_config['max_generation'],
population_size=self.ga_config['population_size'],
population_decay=self.ga_config.get('population_decay', 0.95),
keep_top_ratio=self.ga_config.get('keep_top_ratio', 0.2),
selection_bound=self.ga_config.get('selection_bound', 0.4),
crossover_bound=self.ga_config.get('crossover_bound', 0.4))
else:
raise NotImplementedError(
'Other optimization types are not supported yet')
return algorithm
def main_NN(data, num_samples=15, max_num_epochs=30,
metric = 'loss', mode = 'min',
checkpoint_dir = "C:\\Users\\nhian\\Desktop\\ray_results_12_13\\",
tune_path = "C:/Users/nhian/Dropbox/UCLA MFE/Spring 2020/AFP/ray_results/",
experiment_name = "experiment"+str(date.today()),
trial_name = None):
'''
data = tuple of (X, y) dataset used for training and validation
num_samples = samples to search from search space
max_num_epochs = max number of epochs to train the NN
search over NN hyperspace defined by config
max_num_epochs = max epochs for ASHAScheduler to terminate training
num_samples = num trials
trial_name= current trial name
trial_dir = same as trial name
'''
# config = {
# "num_neurons": tune.choice([16, 32, 64, 128]),
# "num_hidden": tune.choice([2,3,4]),
# "activation_fn" : tune.choice([F.relu, F.leaky_relu]),
# "batch_size": tune.choice([32]),
# "lr": tune.loguniform(1e-4, 1e-1)
# }
bayes_searchspace = {
"num_neurons": Categorical([int(x) for x in 2**np.arange(4,8)]),
"num_hidden": Integer(2, 4, 'uniform'),
"activation_fn" : Categorical([F.relu]),
# "batch_size": Integer(16, 32),#Categorical([int(16), int(32)]),
"lr": Real(1e-4, 1e-2, 'log-uniform')
}
# bayesopt = BayesOptSearch(metric="accuracy", mode="max")
skopt_search = SkOptSearch(space = bayes_searchspace, metric="accuracy", mode="max")
'''
can set metric/mode in scheduler or tune.run
ASHA scheduler can set max_num_epochs
grace_period = min number of iterations before stopping
'''
scheduler = ASHAScheduler(
metric= metric, #loss
mode= mode, #min
time_attr='training_iteration',
max_t=max_num_epochs,
grace_period=10,
reduction_factor=2)
'''
CLIReporter for python console
what to print to console
'''
reporter = CLIReporter(
parameter_columns=["lr", "num_neurons", "num_hidden"],
metric_columns=["loss", "accuracy", "training_iteration"])
#get dataset
x, y = data
input_size = x.shape[1]
#need to register function (trainable) (or use partial and fill out other arguments)
# tune.register_trainable("fc_nn",
# lambda cfg : trainNN(config = cfg, x = x, y = y,
# input_size = input_size, max_num_epochs = max_num_epochs))
def trial_name_string(trial):
return trial_name+str(trial.trial_id)
result = tune.run(
partial(trainNN, x = x, y = y, input_size = input_size, max_num_epochs = max_num_epochs,
checkpoint_dir = checkpoint_dir),
resources_per_trial={"cpu": 3, "gpu": 0},
# config=config,
search_alg = skopt_search,
num_samples=num_samples,
scheduler=scheduler,
reuse_actors = True,
progress_reporter=reporter,
name= experiment_name,
local_dir = tune_path,
trial_name_creator = trial_name_string,
trial_dirname_creator = trial_name_string)
#get best trial
best_trial = result.get_best_trial(metric, mode, "last") #"accuracy" , max
print("Best trial config: {}".format(best_trial.config))
print("Best trial final validation loss: {}".format(
best_trial.last_result["loss"]))
print("Best trial final validation accuracy: {}".format(
best_trial.last_result["accuracy"]))
best_trained_model = net(input_size, best_trial.config["num_neurons"],
best_trial.config["num_hidden"], best_trial.config["activation_fn"])
# best_checkpoint_dir = best_trial.checkpoint.value
# model_state, optimizer_state = torch.load(os.path.join(
# checkpoint_dir, "checkpoint"))
# best_trained_model.load_state_dict(model_state)
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
# if gpus_per_trial > 1:
# best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
return best_trained_model, result
def testConvertSkOpt(self):
from ray.tune.suggest.skopt import SkOptSearch
from skopt.space import Real, Integer
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
SkOptSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = SkOptSearch.convert_search_space(config)
skopt_config = {
"a": [2, 3, 4],
"b/x": Integer(0, 5),
"b/z": Real(1e-4, 1e-2, prior="log-uniform")
}
searcher1 = SkOptSearch(space=converted_config, metric="a", mode="max")
searcher2 = SkOptSearch(space=skopt_config, metric="a", mode="max")
np.random.seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = SkOptSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["a"], [2, 3, 4])
self.assertEqual(trial.config["b"]["y"], 4)
mixed_config = {"a": tune.uniform(5, 6), "b": (8, 9)}
searcher = SkOptSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
help="Specify the out csv filename.",
required=True)
args = parser.parse_args()
# Defining the hyperspace
hyperparameters = [
(0.00001, 0.1), # learning_rate
(0.2, 0.9), # dropout
(10, 100), # epochs
(10, 1000)
] # batch size
space = create_hyperspace(hyperparameters)
# Perform runs and aggregate results
results = []
for section in tqdm(space):
# create a skopt gp minimize object
optimizer = Optimizer(section)
search_algo = SkOptSearch(
optimizer, ['learning_rate', 'dropout', 'epochs', 'batch_size'],
metric='average_res',
mode='max')
analysis = tune.run(multi_train,
search_alg=search_algo,
num_samples=50,
resources_per_trial={'gpu': 1})
results.append(analysis)
all_pt_results = results[0].results_df
for i in range(1, len(results)):
all_pt_results = all_pt_results.append(results[i].results_df)
all_pt_results.to_csv(args.out)
labels,
epsilons=epsilons)
robust_accuracy = 1 - success.cpu().numpy().astype(float).flatten().mean(
axis=-1)
# res test[0] reports the loss from the evaluation, res_test[1] reports the accuracy
tune.report(robust_acc=robust_accuracy)
return robust_accuracy
if __name__ == "__main__":
results = []
for section in tqdm(space):
# create a skopt gp minimize object
optimizer = Optimizer(section)
search_algo = SkOptSearch(
optimizer, ['learning_rate', 'dropout', 'epochs', 'batch_size'],
metric='robust_acc',
mode='max')
# not using a gpu because running on local
analysis = tune.run(mnist_pt_objective,
search_alg=search_algo,
num_samples=20,
resources_per_trial={'gpu': 1})
results.append(analysis)
print(type(results[0]))
all_pt_results = results[0].results_df
for i in range(1, len(results)):
all_pt_results = all_pt_results.append(results[i].results_df)
all_pt_results.to_csv('pt_fool_saltandpepper.csv')
{
"width": 10,
"height": 0,
"activation": "relu" # Activation will be relu
},
{
"width": 15,
"height": -20,
"activation": "tanh" # Activation will be tanh
}
]
known_rewards = [-189, -1144]
algo = SkOptSearch(
# parameter_names=space.keys(), # If you want to set the space
# parameter_ranges=space.values(), # If you want to set the space
points_to_evaluate=previously_run_params,
evaluated_rewards=known_rewards)
algo = ConcurrencyLimiter(algo, max_concurrent=4)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(easy_objective,
metric="mean_loss",
mode="min",
name="skopt_exp_with_warmstart",
search_alg=algo,
scheduler=scheduler,
num_samples=10 if args.smoke_test else 50,
config={
"steps": 100,
def _tune_run(self, config, resources_per_trial):
"""Wrapper to call ``tune.run``. Multiple estimators are generated when
early stopping is possible, whereas a single estimator is
generated when early stopping is not possible.
Args:
config (dict): Configurations such as hyperparameters to run
``tune.run`` on.
resources_per_trial (dict): Resources to use per trial within Ray.
Accepted keys are `cpu`, `gpu` and custom resources, and values
are integers specifying the number of each resource to use.
Returns:
analysis (`ExperimentAnalysis`): Object returned by
`tune.run`.
"""
stop_condition = {"training_iteration": self.max_iters}
if self.early_stopping is not None:
config["estimator_list"] = [
clone(self.estimator) for _ in range(self.n_splits)
]
if hasattr(self.early_stopping, "_max_t_attr"):
# we want to delegate stopping to schedulers which
# support it, but we want it to stop eventually, just in case
# the solution is to make the stop condition very big
stop_condition = {"training_iteration": self.max_iters * 10}
else:
config["estimator_list"] = [self.estimator]
if self.search_optimization == "random":
run_args = dict(scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stop_condition,
num_samples=self.num_samples,
config=config,
fail_fast=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
if isinstance(self.param_distributions, list):
run_args["search_alg"] = RandomListSearcher(
self.param_distributions)
analysis = tune.run(_Trainable, **run_args)
return analysis
elif self.search_optimization == "bayesian":
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
hyperparameter_names, spaces = self._get_skopt_params()
search_algo = SkOptSearch(Optimizer(spaces),
hyperparameter_names,
metric="average_test_score",
**self.search_kwargs)
elif self.search_optimization == "bohb":
from ray.tune.suggest.bohb import TuneBOHB
config_space = self._get_bohb_config_space()
search_algo = TuneBOHB(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
elif self.search_optimization == "optuna":
from ray.tune.suggest.optuna import OptunaSearch
config_space = self._get_optuna_params()
search_algo = OptunaSearch(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
elif self.search_optimization == "hyperopt":
from ray.tune.suggest.hyperopt import HyperOptSearch
config_space = self._get_hyperopt_params()
search_algo = HyperOptSearch(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
if isinstance(self.n_jobs, int) and self.n_jobs > 0:
search_algo = ConcurrencyLimiter(search_algo,
max_concurrent=self.n_jobs)
analysis = tune.run(_Trainable,
search_alg=search_algo,
scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stop_condition,
num_samples=self.num_samples,
config=config,
fail_fast=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
return analysis
def run(exp_ident, exp_name, metric, tune_mode, spec, tune_run_kwargs, use_skopt, skopt_search_mode,
skopt_space, skopt_ref_configs):
spec = sacred_copy(spec)
log_dir = tune_ex.observers[0].dir
ray.init()
def trainable_function(config):
# "config" is passed in by Ray Tune
run_exp(config, log_dir, exp_ident)
if use_skopt:
assert skopt_search_mode in {'min', 'max'}, \
'skopt_search_mode must be "min" or "max", as appropriate for ' \
'the metric being optimised'
assert len(skopt_space) > 0, "was passed an empty skopt_space"
# do some sacred_copy() calls to ensure that we don't accidentally put
# a ReadOnlyDict or ReadOnlyList into our optimizer
skopt_space = sacred_copy(skopt_space)
skopt_search_mode = sacred_copy(skopt_search_mode)
skopt_ref_configs = sacred_copy(skopt_ref_configs)
metric = sacred_copy(metric)
sorted_space = collections.OrderedDict([
(key, value) for key, value in sorted(skopt_space.items())
])
for k, v in list(sorted_space.items()):
# Cast each value in sorted_space to a skopt Dimension object, then
# make the name of the Dimension object match the corresponding
# key. This is the step that converts tuple ranges---like `(1e-3,
# 2.0, 'log-uniform')`---to actual skopt `Space` objects.
try:
new_v = skopt.space.check_dimension(v)
except ValueError:
# Raise actually-informative value error instead
raise ValueError(f"Dimension issue: k:{k} v: {v}")
new_v.name = k
sorted_space[k] = new_v
skopt_optimiser = skopt.optimizer.Optimizer([*sorted_space.values()],
base_estimator='RF')
algo = SkOptSearch(skopt_optimiser,
list(sorted_space.keys()),
metric=metric,
mode=skopt_search_mode,
points_to_evaluate=[[
ref_config_dict[k] for k in sorted_space.keys()
] for ref_config_dict in skopt_ref_configs])
tune_run_kwargs = {
'search_alg': algo,
'scheduler': CheckpointFIFOScheduler(algo),
**tune_run_kwargs,
}
# completely remove 'spec'
if spec:
print("Will ignore everything in 'spec' argument")
spec = {}
tune_run = tune.run(
trainable_function,
name=exp_name,
config=spec,
local_dir=tune_ex.observers[0].dir,
**tune_run_kwargs,
)
best_config = tune_run.get_best_config(metric=metric, mode=tune_mode)
print(f"Best config is: {best_config}")
print("Results available at: ")
print(tune_run._get_trial_paths())
def run_ray_tune(ray_address):
sk_space = collections.OrderedDict()
sk_space['disc_up_per_iter'] = (2, 12) # small values don't work
sk_space['sampler_time_steps'] = (8, 20) # small is okay?
sk_space['sampler_batch_envs'] = (8, 24) # bigger = better?
sk_space['ppo_lr'] = (1e-6, 1e-3, 'log-uniform')
sk_space['ppo_gamma'] = (0.9, 1.0, 'log-uniform')
sk_space['ppo_lambda'] = (0.9, 1.0, 'log-uniform')
sk_space['ppo_ent'] = (1e-6, 1e-4, 'log-uniform')
sk_space['ppo_adv_clip'] = (0.05, 0.2, 'uniform')
sk_space['add_preproc'] = ['LoRes4E', 'LoRes3EA']
# allow us to have smaller batches or run more of them
sk_space['ppo_minibatches'] = [4, 6]
sk_space['ppo_epochs'] = [2, 10]
sk_space['ppo_use_bn'] = [True, False]
sk_space['ppo_aug'] = ['none', 'all', 'crop']
# things I'm commenting out for simplicity:
# sk_space['bc_loss'] = ['0.0', str(int(1e-3)), str(1)]
# sk_space['ppo_use_bn'] = [True, False]
# things that don't matter that much:
# sk_space['omit_noop'] = [True, False] # ???
# sk_space['disc_lr'] = (1e-5, 5e-4, 'log-uniform') # fix to 1e-4
# sk_space['disc_use_act'] = [True, False] # fix to True
# sk_space['disc_all_frames'] = [True, False] # fix to True
# sk_space['disc_replay_mult'] = opt_space.Integer(1, 32, 'log-uniform') # fix to 4 # noqa: E501
# sk_space['ppo_norm_adv'] = [True, False] # fix to False
known_working = {
'disc_up_per_iter': [4, 2],
'sampler_time_steps': [16, 16],
'sampler_batch_envs': [32, 12],
# 'bc_loss': [0.0, 0.0],
'ppo_lr': [2.5e-4, 2e-4],
'ppo_adv_clip': [0.05, 0.1],
'ppo_minibatches': [4, 5],
'ppo_epochs': [4, 6],
'ppo_use_bn': [False, False],
'ppo_aug': ['none', 'none'],
'ppo_gamma': [0.95, 0.9],
'ppo_lambda': [0.95, 0.9],
'ppo_ent': [1e-5, 1.2e-5],
'add_preproc': ['LoRes4E', 'LoRes4E']
# things that I'm removing because they'll take too much time
# 'omit_noop': [True],
# things that don't matter much:
# 'disc_lr': [1e-4],
# 'disc_use_act': [True],
# 'disc_all_frames': [True],
# 'disc_replay_mult': [4],
# 'ppo_norm_adv': [False],
}
for k, v in list(sk_space.items()):
new_v = opt_space.check_dimension(v)
new_v.name = k
sk_space[k] = new_v
sk_optimiser = Optimizer(list(sk_space.values()), base_estimator='GP')
n_known_working, = set(map(len, known_working.values()))
search_alg = SkOptSearch(
sk_optimiser,
sk_space.keys(),
max_concurrent=8, # XXX figure out how to make this configurable
metric='hp_score',
mode='max',
points_to_evaluate=[[known_working[k][i] for k in sk_space]
for i in range(n_known_working)])
if ray_address:
ray.init(redis_address=ray_address)
tune.run(
ray_tune_trial,
search_alg=search_alg,
local_dir='ray-tune-results',
resources_per_trial={"gpu": 0.24},
# this could be 2 days to a week of runs, depending on the env
num_samples=200,
scheduler=CheckpointFIFOScheduler(search_alg))