def testOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch
searcher = OptunaSearch(space=self.config, metric=self.metric_name, mode="max")
self._save(searcher)
searcher = OptunaSearch(space=self.config, metric=self.metric_name, mode="max")
self._restore(searcher)
def set_basic_conf(self):
from optuna.samplers import TPESampler
space = OptunaSearch.convert_search_space({
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100)
})
def cost(space, reporter):
reporter(loss=(space["height"] - 14)**2 - abs(space["width"] - 3))
search_alg = OptunaSearch(
space, sampler=TPESampler(seed=10), metric="loss", mode="min")
return search_alg, cost
def testOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch
from optuna.trial import TrialState
searcher = OptunaSearch(space=self.space,
metric="metric",
mode="max",
points_to_evaluate=[{
self.param_name:
self.valid_value
}],
evaluated_rewards=[1.0])
self.assertGreater(len(searcher._ot_study.trials), 0)
searcher = OptunaSearch(
space=self.space,
metric="metric",
mode="max",
)
point = {
self.param_name: self.valid_value,
}
self.assertEqual(len(searcher._ot_study.trials), 0)
searcher.add_evaluated_point(point,
1.0,
intermediate_values=[0.8, 0.9])
self.assertEqual(len(searcher._ot_study.trials), 1)
self.assertTrue(
searcher._ot_study.trials[-1].state == TrialState.COMPLETE)
searcher.add_evaluated_point(point,
1.0,
intermediate_values=[0.8, 0.9],
error=True)
self.assertEqual(len(searcher._ot_study.trials), 2)
self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.FAIL)
searcher.add_evaluated_point(point,
1.0,
intermediate_values=[0.8, 0.9],
pruned=True)
self.assertEqual(len(searcher._ot_study.trials), 3)
self.assertTrue(
searcher._ot_study.trials[-1].state == TrialState.PRUNED)
def tune_fn():
mlflow.set_experiment(experiment_name=experiment_name)
optuna_search = OptunaSearch(metric="auroc", mode="max")
ax_search = AxSearch(metric="auroc", mode="max")
tune.run(objective,
name="mlflow_gbdt",
num_samples=65,
config={
"num_leaves": tune.randint(5, 95),
"learning_rate": tune.loguniform(1e-4, 1.0),
"n_estimators": tune.randint(100, 100000),
"subsample": tune.loguniform(0.01, 1.0),
"subsample_freq": tune.randint(1, 5),
"objective": "binary",
"reg_alpha": tune.loguniform(1e-4, 1.0),
"reg_lambda": tune.loguniform(1e-4, 1.0),
"tree_learner": "feature",
"feature_sel": 0,
"mlflow": {
"experiment_name": experiment_name,
"tracking_uri": mlflow.get_tracking_uri()
}
},
search_alg=optuna_search)
def set_algorithm(experiment_name, config):
'''
Configure search algorithm.
'''
if args.algorithm == 'hyperopt':
algorithm = HyperOptSearch(points_to_evaluate=best_params)
elif args.algorithm == 'ax':
ax_client = AxClient(enforce_sequential_optimization=False)
ax_client.create_experiment(name=experiment_name,
parameters=config,
objective_name="minimum",
minimize=True)
algorithm = AxSearch(ax_client=ax_client,
points_to_evaluate=best_params)
elif args.algorithm == 'nevergrad':
algorithm = NevergradSearch(
points_to_evaluate=best_params,
optimizer=ng.optimizers.registry["PortfolioDiscreteOnePlusOne"])
elif args.algorithm == 'optuna':
algorithm = OptunaSearch(points_to_evaluate=best_params,
seed=args.seed)
elif args.algorithm == 'pbt':
algorithm = PopulationBasedTraining(
time_attr="training_iteration",
perturbation_interval=args.perturbation,
hyperparam_mutations=config,
synch=True)
elif args.algorithm == 'random':
algorithm = BasicVariantGenerator(max_concurrent=args.jobs)
if args.algorithm not in ['random', 'pbt']:
algorithm = ConcurrencyLimiter(algorithm, max_concurrent=args.jobs)
return algorithm
def set_search_properties(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None,
setting: Optional[Dict] = None,
) -> bool:
metric_changed = mode_changed = False
if metric and self._metric != metric:
metric_changed = True
self._metric = metric
if self._metric_constraints:
# metric modified by lagrange
metric += self.lagrange
# TODO: don't change metric for global search methods that
# can handle constraints already
if mode and self._mode != mode:
mode_changed = True
self._mode = mode
if not self._ls.space:
# the search space can be set only once
if self._gs is not None:
# define-by-run is not supported via set_search_properties
self._gs.set_search_properties(metric, mode, config)
self._gs.space = config
if config:
add_cost_to_space(config, self._ls.init_config,
self._cat_hp_cost)
self._ls.set_search_properties(metric, mode, config)
self._init_search()
else:
if metric_changed or mode_changed:
# reset search when metric or mode changed
self._ls.set_search_properties(metric, mode)
if self._gs is not None:
self._gs = GlobalSearch(
space=self._gs._space,
metric=metric,
mode=mode,
sampler=self._gs._sampler,
)
self._gs.space = self._ls.space
self._init_search()
if setting:
# CFO doesn't need these settings
if "time_budget_s" in setting:
self._time_budget_s = setting[
"time_budget_s"] # budget from now
now = time.time()
self._time_used += now - self._start_time
self._start_time = now
self._set_deadline()
if "metric_target" in setting:
self._metric_target = setting.get("metric_target")
if "num_samples" in setting:
self._num_samples = (setting["num_samples"] +
len(self._result) +
len(self._trial_proposed_by))
return True
def set_basic_conf(self):
from optuna.samplers import TPESampler
space = [
ot_param.suggest_uniform("width", 0, 20),
ot_param.suggest_uniform("height", -100, 100)
]
def cost(space, reporter):
reporter(loss=(space["height"] - 14)**2 - abs(space["width"] - 3))
search_alg = OptunaSearch(
space, sampler=TPESampler(seed=10), metric="loss", mode="min")
return search_alg, cost
def run_tune(method: str, num_samples: int) -> tune.ExperimentAnalysis:
optuna_search = OptunaSearch(metric="mean_accuracy", mode="max")
return tune.run(
trial,
config=methods[method],
num_samples=num_samples,
search_alg=optuna_search,
# resources_per_trial={"gpu": 1, "cpu": 16}, # Using the GPU makes the search process a lot slower on my machine
verbose=1,
metric=
"mean_accuracy", # otherwise I cannot get the dataframe in the end for some reason
mode="max" # See above
)
def init_search_algorithm(search_alg, metric=None, mode=None):
"""Specify a search algorithm and you must pip install it first.
See more details here: https://docs.ray.io/en/master/tune/api_docs/suggestion.html
"""
if search_alg == 'optuna':
assert metric and mode, "Metric and mode cannot be None for optuna."
from ray.tune.suggest.optuna import OptunaSearch
return OptunaSearch(metric=metric, mode=mode)
elif search_alg == 'bayesopt':
assert metric and mode, "Metric and mode cannot be None for bayesian optimization."
from ray.tune.suggest.bayesopt import BayesOptSearch
return BayesOptSearch(metric=metric, mode=mode)
logging.info(f'{search_alg} search is found, run BasicVariantGenerator().')
def testConvergenceOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch
np.random.seed(1)
searcher = OptunaSearch(seed=1)
analysis = self._testConvergence(
searcher,
top=5,
)
# This assertion is much weaker than in the BO case, but TPE
# don't converge too close. It is still unlikely to get to this
# tolerance with random search (5 * 0.1 = 0.5% chance)
assert len(analysis.trials) < 100
assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-1)
def testConvertOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch, param
from optuna.samplers import RandomSampler
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = OptunaSearch.convert_search_space(config)
optuna_config = [
param.suggest_categorical("a", [2, 3, 4]),
param.suggest_int("b/x", 0, 5, 2),
param.suggest_loguniform("b/z", 1e-4, 1e-2)
]
sampler1 = RandomSampler(seed=1234)
searcher1 = OptunaSearch(space=converted_config,
sampler=sampler1,
base_config=config)
sampler2 = RandomSampler(seed=1234)
searcher2 = OptunaSearch(space=optuna_config,
sampler=sampler2,
base_config=config)
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertEqual(config1["b"]["y"], 4)
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = OptunaSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
def run_optuna_tune(smoke_test=False):
algo = OptunaSearch(
space=define_by_run_func, metric="mean_loss", mode="min")
algo = ConcurrencyLimiter(algo, max_concurrent=4)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(
easy_objective,
metric="mean_loss",
mode="min",
search_alg=algo,
scheduler=scheduler,
num_samples=10 if smoke_test else 100,
)
print("Best hyperparameters found were: ", analysis.best_config)
def testOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch
from optuna.samplers import RandomSampler
np.random.seed(1000) # At least one nan, inf, -inf and float
out = tune.run(
_invalid_objective,
search_alg=OptunaSearch(sampler=RandomSampler(seed=1234)),
config=self.config,
mode="max",
num_samples=8,
reuse_actors=False)
best_trial = out.best_trial
self.assertLessEqual(best_trial.config["report"], 2.0)
def run_optuna_tune(smoke_test=False):
algo = OptunaSearch(metric=["loss", "gain"], mode=["min", "max"])
algo = ConcurrencyLimiter(algo, max_concurrent=4)
analysis = tune.run(
easy_objective,
search_alg=algo,
num_samples=10 if smoke_test else 100,
config={
"steps": 100,
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100),
# This is an ignored parameter.
"activation": tune.choice(["relu", "tanh"])
})
print("Best hyperparameters for loss found were: ",
analysis.get_best_config("loss", "min"))
print("Best hyperparameters for gain found were: ",
analysis.get_best_config("gain", "max"))
def init_search_algorithm(search_alg, metric=None, mode=None):
"""Specify a search algorithm and you must pip install it first.
If no search algorithm is specified, the default search algorithm is BasicVariantGenerator.
See more details here: https://docs.ray.io/en/master/tune/api_docs/suggestion.html
Args:
search_alg (str): One of 'basic_variant', 'bayesopt', or 'optuna'.
metric (str): The metric to monitor for early stopping.
mode (str): One of 'min' or 'max' to determine whether to minimize or maximize the metric.
"""
if search_alg == 'optuna':
assert metric and mode, "Metric and mode cannot be None for optuna."
from ray.tune.suggest.optuna import OptunaSearch
return OptunaSearch(metric=metric, mode=mode)
elif search_alg == 'bayesopt':
assert metric and mode, "Metric and mode cannot be None for bayesian optimization."
from ray.tune.suggest.bayesopt import BayesOptSearch
return BayesOptSearch(metric=metric, mode=mode)
logging.info(f'{search_alg} search is found, run BasicVariantGenerator().')
def run_optuna_tune(smoke_test=False):
algo = OptunaSearch()
algo = ConcurrencyLimiter(algo, max_concurrent=4)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(
easy_objective,
metric="mean_loss",
mode="min",
search_alg=algo,
scheduler=scheduler,
num_samples=10 if smoke_test else 100,
config={
"steps": 100,
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100),
# This is an ignored parameter.
"activation": tune.choice(["relu", "tanh"])
})
print("Best hyperparameters found were: ", analysis.best_config)
def testOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch
from optuna.samplers import RandomSampler
np.random.seed(1000)
out = tune.run(_multi_objective,
search_alg=OptunaSearch(
sampler=RandomSampler(seed=1234),
metric=["a", "b", "c"],
mode=["max", "min", "max"],
),
config=self.config,
num_samples=16,
reuse_actors=False)
best_trial_a = out.get_best_trial("a", "max")
self.assertGreaterEqual(best_trial_a.config["a"], 0.8)
best_trial_b = out.get_best_trial("b", "min")
self.assertGreaterEqual(best_trial_b.config["b"], 0.8)
best_trial_c = out.get_best_trial("c", "max")
self.assertGreaterEqual(best_trial_c.config["c"], 0.8)
class BlendSearch(Searcher):
'''class for BlendSearch algorithm
'''
def __init__(self,
metric: Optional[str] = None,
mode: Optional[str] = None,
space: Optional[dict] = None,
points_to_evaluate: Optional[List[Dict]] = None,
cat_hp_cost: Optional[dict] = None,
prune_attr: Optional[str] = None,
min_resource: Optional[float] = None,
max_resource: Optional[float] = None,
reduction_factor: Optional[float] = None,
resources_per_trial: Optional[dict] = None,
global_search_alg: Optional[Searcher] = None,
mem_size=None):
'''Constructor
Args:
metric: A string of the metric name to optimize for.
minimization or maximization.
mode: A string in ['min', 'max'] to specify the objective as
space: A dictionary to specify the search space.
points_to_evaluate: Initial parameter suggestions to be run first.
The first element needs to be a dictionary from a subset of
controlled dimensions to the initial low-cost values.
e.g.,
.. code-block:: python
[{'epochs': 1}]
cat_hp_cost: A dictionary from a subset of categorical dimensions
to the relative cost of each choice.
e.g.,
.. code-block:: python
{'tree_method': [1, 1, 2]}
i.e., the relative cost of the
three choices of 'tree_method' is 1, 1 and 2 respectively.
prune_attr: A string of the attribute used for pruning.
Not necessarily in space.
When prune_attr is in space, it is a hyperparameter, e.g.,
'n_iters', and the best value is unknown.
When prune_attr is not in space, it is a resource dimension,
e.g., 'sample_size', and the peak performance is assumed
to be at the max_resource.
min_resource: A float of the minimal resource to use for the
prune_attr; only valid if prune_attr is not in space.
max_resource: A float of the maximal resource to use for the
prune_attr; only valid if prune_attr is not in space.
reduction_factor: A float of the reduction factor used for
incremental pruning.
resources_per_trial: A dictionary of the resources permitted per
trial, such as 'mem'.
global_search_alg: A Searcher instance as the global search
instance. If omitted, Optuna is used. The following algos have
known issues when used as global_search_alg:
- HyperOptSearch raises exception sometimes
- TuneBOHB has its own scheduler
mem_size: A function to estimate the memory size for a given config.
'''
self._metric, self._mode = metric, mode
if points_to_evaluate: init_config = points_to_evaluate[0]
else: init_config = {}
self._points_to_evaluate = points_to_evaluate
if global_search_alg is not None:
self._gs = global_search_alg
elif getattr(self, '__name__', None) != 'CFO':
self._gs = GlobalSearch(space=space, metric=metric, mode=mode)
else:
self._gs = None
self._ls = LocalSearch(init_config, metric, mode, cat_hp_cost, space,
prune_attr, min_resource, max_resource,
reduction_factor)
self._resources_per_trial = resources_per_trial
self._mem_size = mem_size
self._mem_threshold = resources_per_trial.get(
'mem') if resources_per_trial else None
self._init_search()
def set_search_properties(self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None) -> bool:
if self._ls.space:
if 'time_budget_s' in config:
self._deadline = config.get('time_budget_s') + time.time()
if 'metric_target' in config:
self._metric_target = config.get('metric_target')
else:
if metric: self._metric = metric
if mode: self._mode = mode
self._ls.set_search_properties(metric, mode, config)
if self._gs is not None:
self._gs.set_search_properties(metric, mode, config)
self._init_search()
return True
def _init_search(self):
'''initialize the search
'''
self._metric_target = np.inf * self._ls.metric_op
self._search_thread_pool = {
# id: int -> thread: SearchThread
0: SearchThread(self._ls.mode, self._gs)
}
self._thread_count = 1 # total # threads created
self._init_used = self._ls.init_config is None
self._trial_proposed_by = {} # trial_id: str -> thread_id: int
self._admissible_min = self._ls.normalize(self._ls.init_config)
self._admissible_max = self._admissible_min.copy()
self._result = {} # config_signature: tuple -> result: Dict
self._deadline = np.inf
def save(self, checkpoint_path: str):
save_object = (self._metric_target, self._search_thread_pool,
self._thread_count, self._init_used,
self._trial_proposed_by, self._admissible_min,
self._admissible_max, self._result, self._deadline)
with open(checkpoint_path, "wb") as outputFile:
pickle.dump(save_object, outputFile)
def restore(self, checkpoint_path: str):
with open(checkpoint_path, "rb") as inputFile:
save_object = pickle.load(inputFile)
self._metric_target, self._search_thread_pool, \
self._thread_count, self._init_used, self._trial_proposed_by, \
self._admissible_min, self._admissible_max, self._result, \
self._deadline = save_object
def restore_from_dir(self, checkpoint_dir: str):
super.restore_from_dir(checkpoint_dir)
def on_trial_complete(self,
trial_id: str,
result: Optional[Dict] = None,
error: bool = False):
''' search thread updater and cleaner
'''
thread_id = self._trial_proposed_by.get(trial_id)
if thread_id in self._search_thread_pool:
self._search_thread_pool[thread_id].on_trial_complete(
trial_id, result, error)
del self._trial_proposed_by[trial_id]
# if not thread_id: logger.info(f"result {result}")
if result:
config = {}
for key, value in result.items():
if key.startswith('config/'):
config[key[7:]] = value
if error: # remove from result cache
del self._result[self._ls.config_signature(config)]
else: # add to result cache
self._result[self._ls.config_signature(config)] = result
# update target metric if improved
if (result[self._metric] -
self._metric_target) * self._ls.metric_op < 0:
self._metric_target = result[self._metric]
if thread_id: # from local search
# update admissible region
normalized_config = self._ls.normalize(config)
for key in self._admissible_min:
value = normalized_config[key]
if value > self._admissible_max[key]:
self._admissible_max[key] = value
elif value < self._admissible_min[key]:
self._admissible_min[key] = value
elif self._create_condition(result):
# thread creator
self._search_thread_pool[self._thread_count] = SearchThread(
self._ls.mode,
self._ls.create(config,
result[self._metric],
cost=result["time_total_s"]))
thread_id = self._thread_count
self._thread_count += 1
# cleaner
# logger.info(f"thread {thread_id} in search thread pool="
# f"{thread_id in self._search_thread_pool}")
if thread_id and thread_id in self._search_thread_pool:
# local search thread
self._clean(thread_id)
def _create_condition(self, result: Dict) -> bool:
''' create thread condition
'''
if len(self._search_thread_pool) < 2: return True
obj_median = np.median([
thread.obj_best1
for id, thread in self._search_thread_pool.items() if id
])
return result[self._metric] * self._ls.metric_op < obj_median
def _clean(self, thread_id: int):
''' delete thread and increase admissible region if converged,
merge local threads if they are close
'''
assert thread_id
todelete = set()
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(id, thread_id):
todelete.add(id)
for id in self._search_thread_pool:
if id and id != thread_id:
if self._inferior(thread_id, id):
todelete.add(thread_id)
break
# logger.info(f"thead {thread_id}.converged="
# f"{self._search_thread_pool[thread_id].converged}")
if self._search_thread_pool[thread_id].converged:
todelete.add(thread_id)
for key in self._admissible_min:
self._admissible_max[key] += self._ls.STEPSIZE
self._admissible_min[key] -= self._ls.STEPSIZE
for id in todelete:
del self._search_thread_pool[id]
def _inferior(self, id1: int, id2: int) -> bool:
''' whether thread id1 is inferior to id2
'''
t1 = self._search_thread_pool[id1]
t2 = self._search_thread_pool[id2]
if t1.obj_best1 < t2.obj_best2: return False
elif t1.resource and t1.resource < t2.resource: return False
elif t2.reach(t1): return True
else: return False
def on_trial_result(self, trial_id: str, result: Dict):
if trial_id not in self._trial_proposed_by: return
thread_id = self._trial_proposed_by[trial_id]
if not thread_id in self._search_thread_pool: return
self._search_thread_pool[thread_id].on_trial_result(trial_id, result)
def suggest(self, trial_id: str) -> Optional[Dict]:
''' choose thread, suggest a valid config
'''
if self._init_used and not self._points_to_evaluate:
choice, backup = self._select_thread()
# logger.debug(f"choice={choice}, backup={backup}")
if choice < 0: return None # timeout
self._use_rs = False
config = self._search_thread_pool[choice].suggest(trial_id)
skip = self._should_skip(choice, trial_id, config)
if skip:
if choice:
# logger.info(f"skipping choice={choice}, config={config}")
return None
# use rs
self._use_rs = True
for _, generated in generate_variants(
{'config': self._ls.space}):
config = generated['config']
break
# logger.debug(f"random config {config}")
skip = self._should_skip(choice, trial_id, config)
if skip: return None
# if not choice: logger.info(config)
if choice or backup == choice or self._valid(config):
# LS or valid or no backup choice
self._trial_proposed_by[trial_id] = choice
else: # invalid config proposed by GS
if not self._use_rs:
self._search_thread_pool[choice].on_trial_complete(
trial_id, {}, error=True) # tell GS there is an error
self._use_rs = False
config = self._search_thread_pool[backup].suggest(trial_id)
skip = self._should_skip(backup, trial_id, config)
if skip:
return None
self._trial_proposed_by[trial_id] = backup
choice = backup
# if choice: self._pending.add(choice) # local search thread pending
if not choice:
if self._ls._resource:
# TODO: add resource to config proposed by GS, min or median?
config[self._ls.prune_attr] = self._ls.min_resource
self._result[self._ls.config_signature(config)] = {}
else: # use init config
init_config = self._points_to_evaluate.pop(
0) if self._points_to_evaluate else self._ls.init_config
config = self._ls.complete_config(init_config,
self._admissible_min,
self._admissible_max)
# logger.info(f"reset config to {config}")
config_signature = self._ls.config_signature(config)
result = self._result.get(config_signature)
if result: # tried before
# self.on_trial_complete(trial_id, result)
return None
elif result is None: # not tried before
self._result[config_signature] = {}
else:
return None # running but no result yet
self._init_used = True
# logger.info(f"config={config}")
return config
def _should_skip(self, choice, trial_id, config) -> bool:
''' if config is None or config's result is known or above mem threshold
return True; o.w. return False
'''
if config is None: return True
config_signature = self._ls.config_signature(config)
exists = config_signature in self._result
# check mem constraint
if not exists and self._mem_threshold and self._mem_size(
config) > self._mem_threshold:
self._result[config_signature] = {
self._metric: np.inf * self._ls.metric_op,
'time_total_s': 1
}
exists = True
if exists:
if not self._use_rs:
result = self._result.get(config_signature)
if result:
self._search_thread_pool[choice].on_trial_complete(
trial_id, result, error=False)
if choice:
# local search thread
self._clean(choice)
else:
# tell the thread there is an error
self._search_thread_pool[choice].on_trial_complete(
trial_id, {}, error=True)
return True
return False
def _select_thread(self) -> Tuple:
''' thread selector; use can_suggest to check LS availability
'''
# update priority
min_eci = self._deadline - time.time()
if min_eci <= 0: return -1, -1
max_speed = 0
for thread in self._search_thread_pool.values():
if thread.speed > max_speed: max_speed = thread.speed
for thread in self._search_thread_pool.values():
thread.update_eci(self._metric_target, max_speed)
if thread.eci < min_eci: min_eci = thread.eci
for thread in self._search_thread_pool.values():
thread.update_priority(min_eci)
top_thread_id = backup_thread_id = 0
priority1 = priority2 = self._search_thread_pool[0].priority
# logger.debug(f"priority of thread 0={priority1}")
for thread_id, thread in self._search_thread_pool.items():
# if thread_id:
# logger.debug(
# f"priority of thread {thread_id}={thread.priority}")
# logger.debug(
# f"thread {thread_id}.can_suggest={thread.can_suggest}")
if thread_id and thread.can_suggest:
priority = thread.priority
if priority > priority1:
priority1 = priority
top_thread_id = thread_id
if priority > priority2 or backup_thread_id == 0:
priority2 = priority
backup_thread_id = thread_id
return top_thread_id, backup_thread_id
def _valid(self, config: Dict) -> bool:
''' config validator
'''
for key in self._admissible_min:
if key in config:
value = config[key]
# logger.info(
# f"{key},{value},{self._admissible_min[key]},{self._admissible_max[key]}")
if value < self._admissible_min[
key] or value > self._admissible_max[key]:
return False
return True
def _test_xgboost(method='BlendSearch'):
try:
import ray
except ImportError:
return
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
search_space = {
# You can mix constants with search space objects.
"max_depth": tune.randint(1, 8) if method in [
"BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight": tune.choice([1, 2, 3]),
"subsample": tune.uniform(0.5, 1.0),
"eta": tune.loguniform(1e-4, 1e-1)
}
max_iter = 10
for num_samples in [256]:
time_budget_s = 60 #None
for n_cpu in [8]:
start_time = time.time()
ray.init(num_cpus=n_cpu, num_gpus=0)
if method == 'BlendSearch':
analysis = tune.run(
train_breast_cancer,
init_config={
"max_depth": 1,
"min_child_weight": 3,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
report_intermediate_result=True,
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir='logs/',
num_samples=num_samples*n_cpu,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"max_depth": 1,
"min_child_weight": 3,
}], cat_hp_cost={
"min_child_weight": [6, 3, 2],
})
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples*n_cpu)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(
max_t=max_iter,
grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space, local_dir='logs/',
num_samples=num_samples*n_cpu, time_budget_s=time_budget_s,
scheduler=scheduler, search_alg=algo)
ray.shutdown()
# # Load the best model checkpoint
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss","min","all")
accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def cifar10_main(method='BlendSearch',
num_samples=10,
max_num_epochs=100,
gpus_per_trial=2):
data_dir = os.path.abspath("test/data")
load_data(data_dir) # Download data for all trials before starting the run
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
if method in ['BlendSearch', 'BOHB', 'Optuna']:
config = {
"l1": tune.randint(2, 8),
"l2": tune.randint(2, 8),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs, q=1),
"batch_size": tune.randint(1, 4) #tune.choice([2, 4, 8, 16])
}
else:
config = {
"l1": tune.randint(2, 9),
"l2": tune.randint(2, 9),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1),
"batch_size": tune.randint(1, 5) #tune.choice([2, 4, 8, 16])
}
import ray
time_budget_s = 3600
start_time = time.time()
if method == 'BlendSearch':
result = tune.run(ray.tune.with_parameters(train_cifar,
data_dir=data_dir),
init_config={
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
},
metric="loss",
mode="min",
max_resource=max_num_epochs,
min_resource=1,
report_intermediate_result=True,
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB()
scheduler = HyperBandForBOHB(max_t=max_num_epochs)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
}])
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1)
result = tune.run(tune.with_parameters(train_cifar, data_dir=data_dir),
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
metric="loss",
mode="min",
num_samples=num_samples,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples}")
logger.info(f"time={time.time()-start_time}")
best_trial = result.get_best_trial("loss", "min", "all")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation loss: {}".format(
best_trial.metric_analysis["loss"]["min"]))
logger.info("Best trial final validation accuracy: {}".format(
best_trial.metric_analysis["accuracy"]["max"]))
best_trained_model = Net(2**best_trial.config["l1"],
2**best_trial.config["l2"])
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
if gpus_per_trial > 1:
best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint_path)
best_trained_model.load_state_dict(model_state)
test_acc = _test_accuracy(best_trained_model, device)
logger.info("Best trial test set accuracy: {}".format(test_acc))
def _tune_run(self, config, resources_per_trial):
"""Wrapper to call ``tune.run``. Multiple estimators are generated when
early stopping is possible, whereas a single estimator is
generated when early stopping is not possible.
Args:
config (dict): Configurations such as hyperparameters to run
``tune.run`` on.
resources_per_trial (dict): Resources to use per trial within Ray.
Accepted keys are `cpu`, `gpu` and custom resources, and values
are integers specifying the number of each resource to use.
Returns:
analysis (`ExperimentAnalysis`): Object returned by
`tune.run`.
"""
if self.seed is not None:
random.seed(self.seed)
np.random.seed(self.seed)
trainable = _Trainable
if self.pipeline_auto_early_stop and check_is_pipeline(
self.estimator) and self.early_stopping:
trainable = _PipelineTrainable
max_iter = self.max_iters
if self.early_stopping is not None:
config["estimator_list"] = [
clone(self.estimator) for _ in range(self.n_splits)
]
if hasattr(self.early_stopping, "_max_t_attr"):
# we want to delegate stopping to schedulers which
# support it, but we want it to stop eventually, just in case
# the solution is to make the stop condition very big
max_iter = self.max_iters * 10
else:
config["estimator_list"] = [self.estimator]
stopper = MaximumIterationStopper(max_iter=max_iter)
if self.stopper:
stopper = CombinedStopper(stopper, self.stopper)
run_args = dict(scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stopper,
num_samples=self.n_trials,
config=config,
fail_fast="raise",
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir),
loggers=self.loggers,
time_budget_s=self.time_budget_s)
if self.search_optimization == "random":
if isinstance(self.param_distributions, list):
search_algo = RandomListSearcher(self.param_distributions)
else:
search_algo = BasicVariantGenerator()
run_args["search_alg"] = search_algo
else:
search_space = None
override_search_space = True
if self._is_param_distributions_all_tune_domains():
run_args["config"].update(self.param_distributions)
override_search_space = False
search_kwargs = self.search_kwargs.copy()
search_kwargs.update(metric=self._metric_name, mode="max")
if self.search_optimization == "bayesian":
from ray.tune.suggest.skopt import SkOptSearch
if override_search_space:
search_space = self.param_distributions
search_algo = SkOptSearch(space=search_space, **search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "bohb":
from ray.tune.suggest.bohb import TuneBOHB
if override_search_space:
search_space = self._get_bohb_config_space()
if self.seed:
warnings.warn("'seed' is not implemented for BOHB.")
search_algo = TuneBOHB(space=search_space, **search_kwargs)
# search_algo = TuneBOHB(
# space=search_space, seed=self.seed, **search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "optuna":
from ray.tune.suggest.optuna import OptunaSearch
from optuna.samplers import TPESampler
sampler = TPESampler(seed=self.seed)
if override_search_space:
search_space = self._get_optuna_params()
search_algo = OptunaSearch(space=search_space,
sampler=sampler,
**search_kwargs)
run_args["search_alg"] = search_algo
elif self.search_optimization == "hyperopt":
from ray.tune.suggest.hyperopt import HyperOptSearch
if override_search_space:
search_space = self._get_hyperopt_params()
search_algo = HyperOptSearch(space=search_space,
random_state_seed=self.seed,
**search_kwargs)
run_args["search_alg"] = search_algo
else:
# This should not happen as we validate the input before
# this method. Still, just to be sure, raise an error here.
raise ValueError(
f"Invalid search optimizer: {self.search_optimization}")
if isinstance(self.n_jobs, int) and self.n_jobs > 0 \
and not self.search_optimization == "random":
search_algo = ConcurrencyLimiter(search_algo,
max_concurrent=self.n_jobs)
run_args["search_alg"] = search_algo
with warnings.catch_warnings():
warnings.filterwarnings("ignore",
message="fail_fast='raise' "
"detected.")
analysis = tune.run(trainable, **run_args)
return analysis
def testOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch
from optuna.trial import TrialState
searcher = OptunaSearch(
space=self.space,
metric="metric",
mode="max",
points_to_evaluate=[{self.param_name: self.valid_value}],
evaluated_rewards=[1.0],
)
get_len = lambda s: len(s._ot_study.trials) # noqa E731
self.assertGreater(get_len(searcher), 0)
searcher = OptunaSearch(
space=self.space,
metric="metric",
mode="max",
)
point = {
self.param_name: self.valid_value,
}
self.assertEqual(get_len(searcher), 0)
searcher.add_evaluated_point(point, 1.0, intermediate_values=[0.8, 0.9])
self.assertEqual(get_len(searcher), 1)
self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.COMPLETE)
searcher.add_evaluated_point(
point, 1.0, intermediate_values=[0.8, 0.9], error=True
)
self.assertEqual(get_len(searcher), 2)
self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.FAIL)
searcher.add_evaluated_point(
point, 1.0, intermediate_values=[0.8, 0.9], pruned=True
)
self.assertEqual(get_len(searcher), 3)
self.assertTrue(searcher._ot_study.trials[-1].state == TrialState.PRUNED)
searcher.suggest("1")
searcher = OptunaSearch(
space=self.space,
metric="metric",
mode="max",
)
self.run_add_evaluated_trials(searcher, get_len, get_len)
def dbr_space(trial):
return {self.param_name: trial.suggest_float(self.param_name, 0.0, 5.0)}
dbr_searcher = OptunaSearch(
space=dbr_space,
metric="metric",
mode="max",
)
with self.assertRaises(TypeError):
dbr_searcher.add_evaluated_point(point, 1.0)
# Feed the score back back to Tune.
tune.report(iterations=step, mean_loss=intermediate_score)
time.sleep(0.1)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--smoke-test",
action="store_true",
help="Finish quickly for testing")
args, _ = parser.parse_known_args()
ray.init(configure_logging=False)
algo = OptunaSearch()
algo = ConcurrencyLimiter(algo, max_concurrent=4)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(
easy_objective,
metric="mean_loss",
mode="min",
search_alg=algo,
scheduler=scheduler,
num_samples=10 if args.smoke_test else 100,
config={
"steps": 100,
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100),
# This is an ignored parameter.
"activation": tune.choice(["relu", "tanh"])
def _test_xgboost(method="BlendSearch"):
try:
import ray
except ImportError:
return
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
search_space = {
"max_depth":
tune.randint(1, 9)
if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight":
tune.choice([1, 2, 3]),
"subsample":
tune.uniform(0.5, 1.0),
"eta":
tune.loguniform(1e-4, 1e-1),
}
max_iter = 10
for num_samples in [128]:
time_budget_s = 60
for n_cpu in [2]:
start_time = time.time()
# ray.init(address='auto')
if method == "BlendSearch":
analysis = tune.run(
train_breast_cancer,
config=search_space,
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
scheduler="asha",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif "CFO" == method:
from flaml import CFO
algo = CFO(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "CFOCat" == method:
from flaml.searcher.cfo_cat import CFOCat
algo = CFOCat(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "Dragonfly" == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif "SkOpt" == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif "ZOOpt" == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples * n_cpu)
elif "Ax" == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif "HyperOpt" == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_iter, grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
# # Load the best model checkpoint
# import os
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss", "min", "all")
accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _tune_run(self, config, resources_per_trial):
"""Wrapper to call ``tune.run``. Multiple estimators are generated when
early stopping is possible, whereas a single estimator is
generated when early stopping is not possible.
Args:
config (dict): Configurations such as hyperparameters to run
``tune.run`` on.
resources_per_trial (dict): Resources to use per trial within Ray.
Accepted keys are `cpu`, `gpu` and custom resources, and values
are integers specifying the number of each resource to use.
Returns:
analysis (`ExperimentAnalysis`): Object returned by
`tune.run`.
"""
stop_condition = {"training_iteration": self.max_iters}
if self.early_stopping is not None:
config["estimator_list"] = [
clone(self.estimator) for _ in range(self.n_splits)
]
if hasattr(self.early_stopping, "_max_t_attr"):
# we want to delegate stopping to schedulers which
# support it, but we want it to stop eventually, just in case
# the solution is to make the stop condition very big
stop_condition = {"training_iteration": self.max_iters * 10}
else:
config["estimator_list"] = [self.estimator]
if self.search_optimization == "random":
run_args = dict(scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stop_condition,
num_samples=self.num_samples,
config=config,
fail_fast=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
if isinstance(self.param_distributions, list):
run_args["search_alg"] = RandomListSearcher(
self.param_distributions)
analysis = tune.run(_Trainable, **run_args)
return analysis
elif self.search_optimization == "bayesian":
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
hyperparameter_names, spaces = self._get_skopt_params()
search_algo = SkOptSearch(Optimizer(spaces),
hyperparameter_names,
metric="average_test_score",
**self.search_kwargs)
elif self.search_optimization == "bohb":
from ray.tune.suggest.bohb import TuneBOHB
config_space = self._get_bohb_config_space()
search_algo = TuneBOHB(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
elif self.search_optimization == "optuna":
from ray.tune.suggest.optuna import OptunaSearch
config_space = self._get_optuna_params()
search_algo = OptunaSearch(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
elif self.search_optimization == "hyperopt":
from ray.tune.suggest.hyperopt import HyperOptSearch
config_space = self._get_hyperopt_params()
search_algo = HyperOptSearch(config_space,
metric="average_test_score",
mode="max",
**self.search_kwargs)
if isinstance(self.n_jobs, int) and self.n_jobs > 0:
search_algo = ConcurrencyLimiter(search_algo,
max_concurrent=self.n_jobs)
analysis = tune.run(_Trainable,
search_alg=search_algo,
scheduler=self.early_stopping,
reuse_actors=True,
verbose=self.verbose,
stop=stop_condition,
num_samples=self.num_samples,
config=config,
fail_fast=True,
resources_per_trial=resources_per_trial,
local_dir=os.path.expanduser(self.local_dir))
return analysis
# Feed the score back back to Tune.
tune.report(iterations=step, mean_loss=intermediate_score)
time.sleep(0.1)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"--smoke-test", action="store_true", help="Finish quickly for testing")
args, _ = parser.parse_known_args()
ray.init(configure_logging=False)
space = [
param.suggest_uniform("width", 0, 20),
param.suggest_uniform("height", -100, 100),
# This is an ignored parameter.
param.suggest_categorical("activation", ["relu", "tanh"])
]
config = {
"num_samples": 10 if args.smoke_test else 100,
"config": {
"steps": 100,
}
}
algo = OptunaSearch(space, metric="mean_loss", mode="min")
scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
tune.run(easy_objective, search_alg=algo, scheduler=scheduler, **config)
def testConvertOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch, param
from optuna.samplers import RandomSampler
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
OptunaSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = OptunaSearch.convert_search_space(config)
optuna_config = [
param.suggest_categorical("a", [2, 3, 4]),
param.suggest_int("b/x", 0, 5, 2),
param.suggest_loguniform("b/z", 1e-4, 1e-2)
]
sampler1 = RandomSampler(seed=1234)
searcher1 = OptunaSearch(space=converted_config,
sampler=sampler1,
metric="a",
mode="max")
sampler2 = RandomSampler(seed=1234)
searcher2 = OptunaSearch(space=optuna_config,
sampler=sampler2,
metric="a",
mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = OptunaSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix List and Dict
}
searcher = OptunaSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def __init__(self,
metric: Optional[str] = None,
mode: Optional[str] = None,
space: Optional[dict] = None,
points_to_evaluate: Optional[List[Dict]] = None,
cat_hp_cost: Optional[dict] = None,
prune_attr: Optional[str] = None,
min_resource: Optional[float] = None,
max_resource: Optional[float] = None,
reduction_factor: Optional[float] = None,
resources_per_trial: Optional[dict] = None,
global_search_alg: Optional[Searcher] = None,
mem_size=None):
'''Constructor
Args:
metric: A string of the metric name to optimize for.
minimization or maximization.
mode: A string in ['min', 'max'] to specify the objective as
space: A dictionary to specify the search space.
points_to_evaluate: Initial parameter suggestions to be run first.
The first element needs to be a dictionary from a subset of
controlled dimensions to the initial low-cost values.
e.g.,
.. code-block:: python
[{'epochs': 1}]
cat_hp_cost: A dictionary from a subset of categorical dimensions
to the relative cost of each choice.
e.g.,
.. code-block:: python
{'tree_method': [1, 1, 2]}
i.e., the relative cost of the
three choices of 'tree_method' is 1, 1 and 2 respectively.
prune_attr: A string of the attribute used for pruning.
Not necessarily in space.
When prune_attr is in space, it is a hyperparameter, e.g.,
'n_iters', and the best value is unknown.
When prune_attr is not in space, it is a resource dimension,
e.g., 'sample_size', and the peak performance is assumed
to be at the max_resource.
min_resource: A float of the minimal resource to use for the
prune_attr; only valid if prune_attr is not in space.
max_resource: A float of the maximal resource to use for the
prune_attr; only valid if prune_attr is not in space.
reduction_factor: A float of the reduction factor used for
incremental pruning.
resources_per_trial: A dictionary of the resources permitted per
trial, such as 'mem'.
global_search_alg: A Searcher instance as the global search
instance. If omitted, Optuna is used. The following algos have
known issues when used as global_search_alg:
- HyperOptSearch raises exception sometimes
- TuneBOHB has its own scheduler
mem_size: A function to estimate the memory size for a given config.
'''
self._metric, self._mode = metric, mode
if points_to_evaluate: init_config = points_to_evaluate[0]
else: init_config = {}
self._points_to_evaluate = points_to_evaluate
if global_search_alg is not None:
self._gs = global_search_alg
elif getattr(self, '__name__', None) != 'CFO':
self._gs = GlobalSearch(space=space, metric=metric, mode=mode)
else:
self._gs = None
self._ls = LocalSearch(init_config, metric, mode, cat_hp_cost, space,
prune_attr, min_resource, max_resource,
reduction_factor)
self._resources_per_trial = resources_per_trial
self._mem_size = mem_size
self._mem_threshold = resources_per_trial.get(
'mem') if resources_per_trial else None
self._init_search()
def _test_roberta(method='BlendSearch'):
max_num_epoch = 100
num_samples = -1
time_budget_s = 3600
search_space = {
# You can mix constants with search space objects.
"num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
"learning_rate": flaml.tune.loguniform(1e-5, 3e-5),
"weight_decay": flaml.tune.uniform(0, 0.3),
"per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]),
"seed": flaml.tune.choice([12, 22, 33, 42]),
}
start_time = time.time()
ray.init(num_cpus=4, num_gpus=4)
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import tuneBOHB
algo = tuneBOHB(max_concurrent=4)
scheduler = HyperBandForBOHB(max_t=max_num_epoch)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'BlendSearch' == method:
from flaml import BlendSearch
algo = BlendSearch(
points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch(max_concurrent=3)
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1)
scheduler = None
analysis = ray.tune.run(train_roberta,
metric=HP_METRIC,
mode=MODE,
resources_per_trial={
"gpu": 4,
"cpu": 4
},
config=search_space,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
keep_checkpoints_num=1,
checkpoint_score_attr=HP_METRIC,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
metric = best_trial.metric_analysis[HP_METRIC][MODE]
logger.info(f"method={method}")
logger.info(f"n_trials={len(analysis.trials)}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def main(args):
model = args.model
config_file = args.config_file
log_dir = Path(f"./logs/{model.value}")
log_dir.mkdir(exist_ok=True, parents=True)
data_root = Path(f"./data/processed/{model.value}/IFCNetCore").absolute()
with open("IFCNetCore_Classes.json", "r") as f:
class_names = json.load(f)
if model == Model.MVCNN:
config = {
"batch_size": 64,
"learning_rate": tune.loguniform(1e-5, 1e-2),
"weight_decay": tune.loguniform(1e-4, 1e-2),
"cnn_name": tune.choice(["vgg11", "resnet34", "resnet50"]),
"pretrained": True,
"epochs": 30,
"num_views": 12
}
train_func = partial(train_mvcnn,
data_root=data_root,
class_names=class_names,
eval_on_test=config_file is not None)
elif model == Model.DGCNN:
config = {
"batch_size": 8,
"learning_rate": tune.loguniform(1e-4, 1e-2),
"weight_decay": tune.loguniform(1e-4, 1e-2),
"k": tune.choice([20, 30, 40]),
"embedding_dim": tune.choice([516, 1024, 2048]),
"dropout": 0.5,
"epochs": 250
}
train_func = partial(train_dgcnn,
data_root=data_root,
class_names=class_names,
eval_on_test=config_file is not None)
elif model == Model.MeshNet:
config = {
"batch_size": tune.choice([8, 16, 32]),
"learning_rate": tune.loguniform(1e-4, 1e-2),
"weight_decay": tune.loguniform(1e-4, 1e-2),
"num_kernel": tune.choice([32, 64]),
"sigma": tune.choice([0.1, 0.2, 0.3]),
"aggregation_method": tune.choice(["Concat", "Max", "Average"]),
"epochs": 150
}
train_func = partial(train_meshnet,
data_root=data_root,
class_names=class_names,
eval_on_test=config_file is not None)
if config_file:
with config_file.open("r") as f:
config = json.load(f)
scheduler = ASHAScheduler(max_t=250, grace_period=10)
reporter = CLIReporter(metric_columns=[
"train_balanced_accuracy_score", "val_balanced_accuracy_score",
"training_iteration"
])
result = tune.run(
train_func,
resources_per_trial={
"cpu": 8,
"gpu": 1
},
local_dir=log_dir,
config=config,
mode="max",
metric="val_balanced_accuracy_score",
search_alg=OptunaSearch() if config_file is None else None,
num_samples=20 if config_file is None else 1,
scheduler=scheduler if config_file is None else None,
progress_reporter=reporter)
best_trial = result.get_best_trial("val_balanced_accuracy_score", "max",
"last")
print("Best trial config: {}".format(best_trial.config))
print("Best trial final validation accuracy (balanced): {}".format(
best_trial.last_result["val_balanced_accuracy_score"]))