def __init__(self, config):
super(BOHBProposer, self).__init__(config)
self.tid = 0
self.target = 1 if config[
'target'] == min else -1 # bohb targets for loss only
# BOHB config parameters
set_default_keyvalue("n_iterations", 4, config)
for k, v in BOHB_DEFAULT.items():
set_default_keyvalue(k, v, config)
if not config['min_points_in_model']:
config['min_points_in_model'] = None
# hyperband related parameters
for k, v in SH_DEFAULT.items():
set_default_keyvalue(k, v, config)
# Hyperband related settings - modified from hpbandster/optimizers/bohb.py
self.eta = config['eta']
self.min_budget = config['min_budget']
self.max_budget = config['max_budget']
self.max_SH_iter = -int(
np.log(self.min_budget / self.max_budget) / np.log(self.eta)) + 1
self.budgets = self.max_budget * np.power(
self.eta, -np.linspace(self.max_SH_iter - 1, 0, self.max_SH_iter))
self.n_iterations = config['n_iterations']
self.nSamples = self._get_nSample()
bohb_config = {k: config[k] for k in BOHB_DEFAULT}
configspace = self.create_configspace(config['parameter_config'])
self.config_generator = BOHB(configspace, **bohb_config)
## Based on master.py
self.iterations = []
self.running_jobs = {}
def train_kde(self, result, config_space):
cg = BohbConfigGenerator(config_space)
results_for_budget = dict()
build_model_jobs = dict()
id2conf = result.get_id2config_mapping()
for id in id2conf:
for r in result.get_runs_by_id(id):
j = Job(id, config=id2conf[id]['config'], budget=r.budget)
if r.loss is None:
r.loss = float('inf')
if r.info is None:
r.info = dict()
j.result = {'loss': r.loss, 'info': r.info}
j.error_logs = r.error_logs
if r.budget not in results_for_budget:
results_for_budget[r.budget] = list()
results_for_budget[r.budget].append(j)
if r.loss is not None and r.budget not in build_model_jobs:
build_model_jobs[r.budget] = j
continue
cg.new_result(j, update_model=False)
for j in build_model_jobs.values():
cg.new_result(j, update_model=True)
good_kdes = [m["good"] for b, m in sorted(cg.kde_models.items())]
bad_kdes = [m["bad"] for b, m in sorted(cg.kde_models.items())]
budgets = sorted(cg.kde_models.keys())
return good_kdes, bad_kdes, budgets
def setup_bohb(self):
from hpbandster.optimizers.config_generators.bohb import BOHB
if self._mode == "max":
self._metric_op = -1.
elif self._mode == "min":
self._metric_op = 1.
bohb_config = self._bohb_config or {}
self.bohber = BOHB(self._space, **bohb_config)
def _get_likelihood_matrix(self, train_data_good, train_data_bad,
kdes_good, kdes_bad,
kde_configspaces): # datapoints x models
likelihood_matrix = np.empty(
(train_data_good.shape[0] + train_data_bad.shape[0],
len(kdes_good)))
for i, (good_kde, bad_kde, kde_configspace) in enumerate(
zip(kdes_good, kdes_bad, kde_configspaces)):
train_data_good_compatible = train_data_good
train_data_bad_compatible = train_data_bad
# compute likelihood of kde given observation
pdf = KDEMultivariate.pdf # leave_given_out_pdf
if not self.warmstarted_model.is_current_kde(i):
imputer = BOHB(kde_configspace).impute_conditional_data
train_data_good_compatible = make_vector_compatible(
train_data_good, self.configspace, kde_configspace,
imputer)
train_data_bad_compatible = make_vector_compatible(
train_data_bad, self.configspace, kde_configspace, imputer)
pdf = KDEMultivariate.pdf
good_kde_likelihoods = np.nan_to_num(
pdf(good_kde, train_data_good_compatible))
bad_kde_likelihoods = np.nan_to_num(
pdf(bad_kde, train_data_bad_compatible))
likelihood_matrix[:, i] = np.append(good_kde_likelihoods,
bad_kde_likelihoods)
return likelihood_matrix
def _setup_bohb(self):
from hpbandster.optimizers.config_generators.bohb import BOHB
if self._metric is None and self._mode:
# If only a mode was passed, use anonymous metric
self._metric = DEFAULT_METRIC
if self._mode == "max":
self._metric_op = -1.
elif self._mode == "min":
self._metric_op = 1.
if self._seed is not None:
self._space.seed(self._seed)
bohb_config = self._bohb_config or {}
self.bohber = BOHB(self._space, **bohb_config)
def __init__(self,
space,
bohb_config=None,
max_concurrent=10,
metric="neg_mean_loss",
mode="max"):
from hpbandster.optimizers.config_generators.bohb import BOHB
assert BOHB is not None, "HpBandSter must be installed!"
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!"
self._max_concurrent = max_concurrent
self.trial_to_params = {}
self.running = set()
self.paused = set()
self._metric = metric
if mode == "max":
self._metric_op = -1.
elif mode == "min":
self._metric_op = 1.
bohb_config = bohb_config or {}
self.bohber = BOHB(space, **bohb_config)
super(TuneBOHB, self).__init__(metric=self._metric, mode=mode)
def __init__(self,
configspace=None,
eta=3,
min_budget=0.01,
max_budget=1,
min_points_in_model=None,
top_n_percent=15,
num_samples=64,
random_fraction=1 / 3,
bandwidth_factor=3,
min_bandwidth=1e-3,
**kwargs):
# TODO: Proper check for ConfigSpace object!
if configspace is None:
raise ValueError("You have to provide a valid CofigSpace object")
cg = BOHB(configspace=configspace,
min_points_in_model=min_points_in_model,
top_n_percent=top_n_percent,
num_samples=num_samples,
random_fraction=random_fraction,
bandwidth_factor=bandwidth_factor,
min_bandwidth=min_bandwidth)
super().__init__(config_generator=cg, **kwargs)
# Hyperband related stuff
self.eta = eta
self.min_budget = min_budget
self.max_budget = max_budget
# precompute some HB stuff
self.max_SH_iter = -int(
np.log(min_budget / max_budget) / np.log(eta)) + 1
self.budgets = max_budget * np.power(
eta, -np.linspace(self.max_SH_iter - 1, 0, self.max_SH_iter))
self.config.update({
'eta': eta,
'min_budget': min_budget,
'max_budget': max_budget,
'budgets': self.budgets,
'max_SH_iter': self.max_SH_iter,
'min_points_in_model': min_points_in_model,
'top_n_percent': top_n_percent,
'num_samples': num_samples,
'random_fraction': random_fraction,
'bandwidth_factor': bandwidth_factor,
'min_bandwidth': min_bandwidth
})
def _setup(self):
self.max_sh_iter = (
-int(np.log(self.min_budget / self.max_budget) / np.log(self.eta))
+ 1)
self.budgets = self.max_budget * np.power(
self.eta, -np.linspace(self.max_sh_iter - 1, 0, self.max_sh_iter))
self.bohb = CG_BOHB( # pylint: disable=attribute-defined-outside-init
configspace=convert_space(self.space),
min_points_in_model=self.min_points_in_model,
top_n_percent=self.top_n_percent,
num_samples=self.num_samples,
random_fraction=self.random_fraction,
bandwidth_factor=self.bandwidth_factor,
min_bandwidth=self.min_bandwidth,
)
self.bohb.configspace.seed(self.seed)
def test_imputation_conditional_spaces(self):
bohb = BOHB(self.configspace, random_fraction=0)
raw_array = []
for i in range(128):
config = self.configspace.sample_configuration()
raw_array.append(config.get_array())
imputed_array = bohb.impute_conditional_data(np.array(raw_array))
self.assertFalse(np.any(np.isnan(imputed_array)))
job = Job(i, budget=1, config=config)
job.result = {'loss': np.random.rand(), 'info': {}}
bohb.new_result(job)
for j in range(64):
conf, info = bohb.get_config(1)
self.assertTrue(info['model_based_pick'])
class BOHBProposer(AbstractProposer):
def __init__(self, config):
super(BOHBProposer, self).__init__(config)
self.tid = 0
self.target = 1 if config[
'target'] == min else -1 # bohb targets for loss only
# BOHB config parameters
set_default_keyvalue("n_iterations", 4, config)
for k, v in BOHB_DEFAULT.items():
set_default_keyvalue(k, v, config)
if not config['min_points_in_model']:
config['min_points_in_model'] = None
# hyperband related parameters
for k, v in SH_DEFAULT.items():
set_default_keyvalue(k, v, config)
# Hyperband related settings - modified from hpbandster/optimizers/bohb.py
self.eta = config['eta']
self.min_budget = config['min_budget']
self.max_budget = config['max_budget']
self.max_SH_iter = -int(
np.log(self.min_budget / self.max_budget) / np.log(self.eta)) + 1
self.budgets = self.max_budget * np.power(
self.eta, -np.linspace(self.max_SH_iter - 1, 0, self.max_SH_iter))
self.n_iterations = config['n_iterations']
self.nSamples = self._get_nSample()
bohb_config = {k: config[k] for k in BOHB_DEFAULT}
configspace = self.create_configspace(config['parameter_config'])
self.config_generator = BOHB(configspace, **bohb_config)
## Based on master.py
self.iterations = []
self.running_jobs = {}
def get_param(self):
"""
Get the next hyperparameter values, return None when experiment is finished.
:return: hyperparameters in dictionary
"""
while True:
next_run = None
for i in self.active_iterations():
next_run = self.iterations[i].get_next_run()
if next_run is not None:
break
if next_run is not None:
logger.debug("new hyperparameters %s" % (next_run, ))
break
else:
if self.n_iterations > 0:
logger.debug("create new iteration for %d" %
self.n_iterations)
self.iterations.append(
self.get_next_iteration(len(self.iterations)))
self.n_iterations -= 1
else:
self.finished = True
return None
config_id, config, budget = next_run
job = Job(config_id, config=config, budget=budget)
job.time_it("started")
self.running_jobs[self.tid] = job
config = config.copy()
config['tid'] = self.tid
config['n_iterations'] = budget # for job execution
self.tid += 1
return config
def update(self, score, job):
"""
Wrap result and transfer to HpBandSter
:param score: result
:param job: job contains job id for configuration matching
"""
i = job.config['tid']
job = self.running_jobs[i]
job.time_it("finished")
job.result = {'loss': score * self.target}
self.iterations[job.id[0]].register_result(job)
self.config_generator.new_result(job)
del self.running_jobs[i]
def failed(self, job):
"""
Failed jobs unsupported by BOHB Proposer.
:param job: Failed job, containing job id
:type job: Job
"""
super().failed(job)
raise NotImplementedError("BOHBProposer does not support failed jobs")
def get_next_iteration(self, iteration, iteration_kwargs={}):
""" Copied from https://github.com/automl/HpBandSter/blob/master/hpbandster/optimizers/bohb.py
"""
s = self.max_SH_iter - 1 - (iteration % self.max_SH_iter)
n0 = int(np.floor(self.max_SH_iter / (s + 1)) * self.eta**s)
ns = [max(int(n0 * (self.eta**(-i))), 1) for i in range(s + 1)]
return (SuccessiveHalving(
HPB_iter=iteration,
num_configs=ns,
budgets=self.budgets[(-s - 1):],
config_sampler=self.config_generator.get_config,
**iteration_kwargs))
def _get_nSample(self):
nSamples = 0
for iteration in range(self.n_iterations):
s = self.max_SH_iter - 1 - (iteration % self.max_SH_iter)
n0 = int(np.floor(self.max_SH_iter / (s + 1)) * self.eta**s)
ns = [max(int(n0 * (self.eta**(-i))), 1) for i in range(s + 1)]
nSamples += sum(ns)
logger.debug("total exp %d:" % nSamples)
return nSamples
@staticmethod
def create_configspace(parameter_config):
"""
Wrap the Worker's get_configspace() function for HpBandSter interface
"""
cs = CS.ConfigurationSpace()
params = []
for config in parameter_config:
p = AbstractProposer.parse_param_config(config)
if p['type'] == 'choice':
param = CS.CategoricalHyperparameter(p['name'],
choices=p['range'])
else: # for int or float
param = dict(name=p['name'])
param['lower'], param['upper'] = min(p['range']), max(
p['range'])
if p['type'] == 'int':
param = CS.UniformIntegerHyperparameter(**param)
else:
param = CS.UniformFloatHyperparameter(**param)
params.append(param)
cs.add_hyperparameters(params)
return cs
@staticmethod
def setup_config(): # pragma: no cover
config = dict()
for k, v in BOHB_DEFAULT.items():
if k == "min_points_in_model":
config[k] = input("%s [%s]:" % (k, v))
if config[k]:
config[k] = int(config[k])
else:
config[k] = type(v)(input("%s [%s]:" % (k, v)) or v)
for k, v in SH_DEFAULT.items():
config[k] = type(v)(input("%s [%s]:" % (k, v)) or v)
config.update(AbstractProposer.setup_config())
return config
def active_iterations(self): # pragma: no cover
"""
Based on :func:`hpbandster.core.master.Master.active_iterations`.
"""
return list(
filter(lambda idx: not self.iterations[idx].is_finished,
range(len(self.iterations))))
def save(self, path):
msg = "Save and restore not supported yet"
logger.fatal(msg)
raise NotImplementedError(msg)
def reload(self, path):
msg = "Save and restore not supported yet"
logger.fatal(msg)
raise NotImplementedError(msg)
class TuneBOHB(Searcher):
"""BOHB suggestion component.
Requires HpBandSter and ConfigSpace to be installed. You can install
HpBandSter and ConfigSpace with: ``pip install hpbandster ConfigSpace``.
This should be used in conjunction with HyperBandForBOHB.
Args:
space (ConfigurationSpace): Continuous ConfigSpace search space.
Parameters will be sampled from this space which will be used
to run trials.
bohb_config (dict): configuration for HpBandSter BOHB algorithm
max_concurrent (int): Deprecated. Use
``tune.suggest.ConcurrencyLimiter()``.
metric (str): The training result objective value attribute. If None
but a mode was passed, the anonymous metric `_metric` will be used
per default.
mode (str): One of {min, max}. Determines whether objective is
minimizing or maximizing the metric attribute.
points_to_evaluate (list): Initial parameter suggestions to be run
first. This is for when you already have some good parameters
you want to run first to help the algorithm make better suggestions
for future parameters. Needs to be a list of dicts containing the
configurations.
seed (int): Optional random seed to initialize the random number
generator. Setting this should lead to identical initial
configurations at each run.
Tune automatically converts search spaces to TuneBOHB's format:
.. code-block:: python
config = {
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100),
"activation": tune.choice(["relu", "tanh"])
}
algo = TuneBOHB(metric="mean_loss", mode="min")
bohb = HyperBandForBOHB(
time_attr="training_iteration",
metric="mean_loss",
mode="min",
max_t=100)
run(my_trainable, config=config, scheduler=bohb, search_alg=algo)
If you would like to pass the search space manually, the code would
look like this:
.. code-block:: python
import ConfigSpace as CS
config_space = CS.ConfigurationSpace()
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter("width", lower=0, upper=20))
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
config_space.add_hyperparameter(
CS.CategoricalHyperparameter(
name="activation", choices=["relu", "tanh"]))
algo = TuneBOHB(
config_space, metric="mean_loss", mode="min")
bohb = HyperBandForBOHB(
time_attr="training_iteration",
metric="mean_loss",
mode="min",
max_t=100)
run(my_trainable, scheduler=bohb, search_alg=algo)
"""
def __init__(
self,
space: Optional[Union[Dict,
"ConfigSpace.ConfigurationSpace"]] = None,
bohb_config: Optional[Dict] = None,
max_concurrent: Optional[int] = None,
metric: Optional[str] = None,
mode: Optional[str] = None,
points_to_evaluate: Optional[List[Dict]] = None,
seed: Optional[int] = None):
assert BOHB is not None, """HpBandSter must be installed!
You can install HpBandSter with the command:
`pip install hpbandster ConfigSpace`."""
if mode:
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
self._max_concurrent = max_concurrent
self.trial_to_params = {}
self._metric = metric
self._bohb_config = bohb_config
if isinstance(space, dict) and space:
resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
if domain_vars or grid_vars:
logger.warning(
UNRESOLVED_SEARCH_SPACE.format(par="space",
cls=type(self)))
space = self.convert_search_space(space)
self._space = space
self._seed = seed
self._points_to_evaluate = points_to_evaluate
super(TuneBOHB, self).__init__(metric=self._metric,
mode=mode,
max_concurrent=max_concurrent)
if self._space:
self._setup_bohb()
def _setup_bohb(self):
from hpbandster.optimizers.config_generators.bohb import BOHB
if self._metric is None and self._mode:
# If only a mode was passed, use anonymous metric
self._metric = DEFAULT_METRIC
if self._mode == "max":
self._metric_op = -1.
elif self._mode == "min":
self._metric_op = 1.
if self._seed is not None:
self._space.seed(self._seed)
bohb_config = self._bohb_config or {}
self.bohber = BOHB(self._space, **bohb_config)
def set_search_properties(self, metric: Optional[str], mode: Optional[str],
config: Dict, **spec) -> bool:
if self._space:
return False
space = self.convert_search_space(config)
self._space = space
if metric:
self._metric = metric
if mode:
self._mode = mode
self._setup_bohb()
return True
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._space:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self._points_to_evaluate:
config = self._points_to_evaluate.pop(0)
else:
# This parameter is not used in hpbandster implementation.
config, _ = self.bohber.get_config(None)
self.trial_to_params[trial_id] = copy.deepcopy(config)
return unflatten_list_dict(config)
def on_trial_result(self, trial_id: str, result: Dict):
if "hyperband_info" not in result:
logger.warning("BOHB Info not detected in result. Are you using "
"HyperBandForBOHB as a scheduler?")
elif "budget" in result.get("hyperband_info", {}):
hbs_wrapper = self.to_wrapper(trial_id, result)
self.bohber.new_result(hbs_wrapper)
def on_trial_complete(self,
trial_id: str,
result: Optional[Dict] = None,
error: bool = False):
del self.trial_to_params[trial_id]
def to_wrapper(self, trial_id: str, result: Dict) -> _BOHBJobWrapper:
return _BOHBJobWrapper(self._metric_op * result[self.metric],
result["hyperband_info"]["budget"],
self.trial_to_params[trial_id])
@staticmethod
def convert_search_space(spec: Dict) -> "ConfigSpace.ConfigurationSpace":
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
if grid_vars:
raise ValueError(
"Grid search parameters cannot be automatically converted "
"to a TuneBOHB search space.")
# Flatten and resolve again after checking for grid search.
spec = flatten_dict(spec, prevent_delimiter=True)
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
def resolve_value(
par: str,
domain: Domain) -> ConfigSpace.hyperparameters.Hyperparameter:
quantize = None
sampler = domain.get_sampler()
if isinstance(sampler, Quantized):
quantize = sampler.q
sampler = sampler.sampler
if isinstance(domain, Float):
if isinstance(sampler, LogUniform):
lower = domain.lower
upper = domain.upper
if quantize:
lower = math.ceil(domain.lower / quantize) * quantize
upper = math.floor(domain.upper / quantize) * quantize
return ConfigSpace.UniformFloatHyperparameter(par,
lower=lower,
upper=upper,
q=quantize,
log=True)
elif isinstance(sampler, Uniform):
lower = domain.lower
upper = domain.upper
if quantize:
lower = math.ceil(domain.lower / quantize) * quantize
upper = math.floor(domain.upper / quantize) * quantize
return ConfigSpace.UniformFloatHyperparameter(par,
lower=lower,
upper=upper,
q=quantize,
log=False)
elif isinstance(sampler, Normal):
return ConfigSpace.hyperparameters.\
NormalFloatHyperparameter(
par,
mu=sampler.mean,
sigma=sampler.sd,
q=quantize,
log=False)
elif isinstance(domain, Integer):
if isinstance(sampler, LogUniform):
lower = domain.lower
upper = domain.upper
if quantize:
lower = math.ceil(domain.lower / quantize) * quantize
upper = math.floor(domain.upper / quantize) * quantize
else:
# Tune search space integers are exclusive
upper -= 1
return ConfigSpace.UniformIntegerHyperparameter(
par, lower=lower, upper=upper, q=quantize, log=True)
elif isinstance(sampler, Uniform):
lower = domain.lower
upper = domain.upper
if quantize:
lower = math.ceil(domain.lower / quantize) * quantize
upper = math.floor(domain.upper / quantize) * quantize
else:
# Tune search space integers are exclusive
upper -= 1
return ConfigSpace.UniformIntegerHyperparameter(
par, lower=lower, upper=upper, q=quantize, log=False)
elif isinstance(domain, Categorical):
if isinstance(sampler, Uniform):
return ConfigSpace.CategoricalHyperparameter(
par, choices=domain.categories)
raise ValueError("TuneBOHB does not support parameters of type "
"`{}` with samplers of type `{}`".format(
type(domain).__name__,
type(domain.sampler).__name__))
cs = ConfigSpace.ConfigurationSpace()
for path, domain in domain_vars:
par = "/".join(str(p) for p in path)
value = resolve_value(par, domain)
cs.add_hyperparameter(value)
return cs
def save(self, checkpoint_path: str):
save_object = self.__dict__
with open(checkpoint_path, "wb") as outputFile:
cloudpickle.dump(save_object, outputFile)
def restore(self, checkpoint_path: str):
with open(checkpoint_path, "rb") as inputFile:
save_object = cloudpickle.load(inputFile)
self.__dict__.update(save_object)
class TuneBOHB(Searcher):
"""BOHB suggestion component.
Requires HpBandSter and ConfigSpace to be installed. You can install
HpBandSter and ConfigSpace with: ``pip install hpbandster ConfigSpace``.
This should be used in conjunction with HyperBandForBOHB.
Args:
space (ConfigurationSpace): Continuous ConfigSpace search space.
Parameters will be sampled from this space which will be used
to run trials.
bohb_config (dict): configuration for HpBandSter BOHB algorithm
max_concurrent (int): Number of maximum concurrent trials. Defaults
to 10.
metric (str): The training result objective value attribute.
mode (str): One of {min, max}. Determines whether objective is
minimizing or maximizing the metric attribute.
Tune automatically converts search spaces to TuneBOHB's format:
.. code-block:: python
config = {
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100),
"activation": tune.choice(["relu", "tanh"])
}
algo = TuneBOHB(max_concurrent=4, metric="mean_loss", mode="min")
bohb = HyperBandForBOHB(
time_attr="training_iteration",
metric="mean_loss",
mode="min",
max_t=100)
run(my_trainable, config=config, scheduler=bohb, search_alg=algo)
If you would like to pass the search space manually, the code would
look like this:
.. code-block:: python
import ConfigSpace as CS
config_space = CS.ConfigurationSpace()
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter("width", lower=0, upper=20))
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
config_space.add_hyperparameter(
CS.CategoricalHyperparameter(
name="activation", choices=["relu", "tanh"]))
algo = TuneBOHB(
config_space, max_concurrent=4, metric="mean_loss", mode="min")
bohb = HyperBandForBOHB(
time_attr="training_iteration",
metric="mean_loss",
mode="min",
max_t=100)
run(my_trainable, scheduler=bohb, search_alg=algo)
"""
def __init__(self,
space=None,
bohb_config=None,
max_concurrent=10,
metric=None,
mode=None):
from hpbandster.optimizers.config_generators.bohb import BOHB
assert BOHB is not None, "HpBandSter must be installed!"
if mode:
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
self._max_concurrent = max_concurrent
self.trial_to_params = {}
self.running = set()
self.paused = set()
self._metric = metric
self._bohb_config = bohb_config
self._space = space
super(TuneBOHB, self).__init__(metric=self._metric, mode=mode)
if self._space:
self.setup_bohb()
def setup_bohb(self):
from hpbandster.optimizers.config_generators.bohb import BOHB
if self._mode == "max":
self._metric_op = -1.
elif self._mode == "min":
self._metric_op = 1.
bohb_config = self._bohb_config or {}
self.bohber = BOHB(self._space, **bohb_config)
def set_search_properties(self, metric, mode, config):
if self._space:
return False
space = self.convert_search_space(config)
self._space = space
if metric:
self._metric = metric
if mode:
self._mode = mode
self.setup_bohb()
return True
def suggest(self, trial_id):
if not self._space:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
if len(self.running) < self._max_concurrent:
# This parameter is not used in hpbandster implementation.
config, info = self.bohber.get_config(None)
self.trial_to_params[trial_id] = copy.deepcopy(config)
self.running.add(trial_id)
return unflatten_dict(config)
return None
def on_trial_result(self, trial_id, result):
if trial_id not in self.paused:
self.running.add(trial_id)
if "hyperband_info" not in result:
logger.warning("BOHB Info not detected in result. Are you using "
"HyperBandForBOHB as a scheduler?")
elif "budget" in result.get("hyperband_info", {}):
hbs_wrapper = self.to_wrapper(trial_id, result)
self.bohber.new_result(hbs_wrapper)
def on_trial_complete(self, trial_id, result=None, error=False):
del self.trial_to_params[trial_id]
if trial_id in self.paused:
self.paused.remove(trial_id)
if trial_id in self.running:
self.running.remove(trial_id)
def to_wrapper(self, trial_id, result):
return _BOHBJobWrapper(self._metric_op * result[self.metric],
result["hyperband_info"]["budget"],
self.trial_to_params[trial_id])
def on_pause(self, trial_id):
self.paused.add(trial_id)
self.running.remove(trial_id)
def on_unpause(self, trial_id):
self.paused.remove(trial_id)
self.running.add(trial_id)
@staticmethod
def convert_search_space(spec: Dict):
spec = flatten_dict(spec, prevent_delimiter=True)
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
if grid_vars:
raise ValueError(
"Grid search parameters cannot be automatically converted "
"to a TuneBOHB search space.")
def resolve_value(par, domain):
quantize = None
sampler = domain.get_sampler()
if isinstance(sampler, Quantized):
quantize = sampler.q
sampler = sampler.sampler
if isinstance(domain, Float):
if isinstance(sampler, LogUniform):
lower = domain.lower
upper = domain.upper
if quantize:
lower = math.ceil(domain.lower / quantize) * quantize
upper = math.floor(domain.upper / quantize) * quantize
return ConfigSpace.UniformFloatHyperparameter(par,
lower=lower,
upper=upper,
q=quantize,
log=True)
elif isinstance(sampler, Uniform):
lower = domain.lower
upper = domain.upper
if quantize:
lower = math.ceil(domain.lower / quantize) * quantize
upper = math.floor(domain.upper / quantize) * quantize
return ConfigSpace.UniformFloatHyperparameter(par,
lower=lower,
upper=upper,
q=quantize,
log=False)
elif isinstance(sampler, Normal):
return ConfigSpace.NormalFloatHyperparameter(
par,
mu=sampler.mean,
sigma=sampler.sd,
q=quantize,
log=False)
elif isinstance(domain, Integer):
if isinstance(sampler, Uniform):
lower = domain.lower
upper = domain.upper
if quantize:
lower = math.ceil(domain.lower / quantize) * quantize
upper = math.floor(domain.upper / quantize) * quantize
return ConfigSpace.UniformIntegerHyperparameter(
par, lower=lower, upper=upper, q=quantize, log=False)
elif isinstance(domain, Categorical):
if isinstance(sampler, Uniform):
return ConfigSpace.CategoricalHyperparameter(
par, choices=domain.categories)
raise ValueError("TuneBOHB does not support parameters of type "
"`{}` with samplers of type `{}`".format(
type(domain).__name__,
type(domain.sampler).__name__))
cs = ConfigSpace.ConfigurationSpace()
for path, domain in domain_vars:
par = "/".join(path)
value = resolve_value(par, domain)
cs.add_hyperparameter(value)
return cs
class BOHB(BaseAlgorithm):
"""Bayesian Optimization with HyperBand
This class is a wrapper around the librairy HpBandSter:
https://github.com/automl/HpBandSter.
For more information on the algorithm,
see original paper at https://arxiv.org/abs/1807.01774.
Falkner, Stefan, Aaron Klein, and Frank Hutter. "BOHB: Robust and efficient hyperparameter
optimization at scale." In International Conference on Machine Learning, pp. 1437-1446. PMLR,
2018.
Parameters
----------
space: `orion.algo.space.Space`
Optimisation space with priors for each dimension.
seed: None, int or sequence of int
Seed for the random number generator used to sample new trials.
Default: ``None``
min_points_in_model: int
Number of observations to start building a KDE. If ``None``, uses number
of dimensions in the search space + 1. Default: ``None``
top_n_percent: int
Percentage ( between 1 and 99) of the observations that are considered good. Default: 15
num_samples: int
Number of samples to optimize Expected Improvement. Default: 64
random_fraction: float
Fraction of purely random configurations that are sampled from the
prior without the model. Default: 1/3
bandwidth_factor: float
To encourage diversity, the points proposed to optimize EI, are sampled
from a 'widened' KDE where the bandwidth is multiplied by this factor. Default: 3
min_bandwidth: float
To keep diversity, even when all (good) samples have the same value
for one of the parameters, a minimum bandwidth is used instead of
zero. Default: 1e-3
parallel_strategy: dict or None, optional
The configuration of a parallel strategy to use for pending trials or broken trials.
Default is a MaxParallelStrategy for broken trials and NoParallelStrategy for pending
trials.
"""
requires_type = None
requires_dist = None
requires_shape = "flattened"
def __init__(
self,
space,
seed=None,
min_points_in_model=None,
top_n_percent=15,
num_samples=64,
random_fraction=1 / 3,
bandwidth_factor=3,
min_bandwidth=1e-3,
parallel_strategy=None,
): # pylint: disable=too-many-arguments
if parallel_strategy is None:
parallel_strategy = {
"of_type": "StatusBasedParallelStrategy",
"strategy_configs": {
"broken": {
"of_type": "MaxParallelStrategy",
},
},
}
self.strategy = strategy_factory.create(**parallel_strategy)
super().__init__(
space,
seed=seed,
min_points_in_model=min_points_in_model,
top_n_percent=top_n_percent,
num_samples=num_samples,
random_fraction=random_fraction,
bandwidth_factor=bandwidth_factor,
min_bandwidth=min_bandwidth,
parallel_strategy=parallel_strategy,
)
self.trial_meta = {}
self.trial_results = {}
self.iteration = 0
self.iterations = []
fidelity_index = self.fidelity_index
if fidelity_index is None:
raise RuntimeError(SPACE_ERROR)
fidelity_dim = self.space[fidelity_index]
fidelity_dim: Fidelity = space[self.fidelity_index]
# NOTE: This isn't a Fidelity, it's a TransformedDimension<Fidelity>
from orion.core.worker.transformer import TransformedDimension
# NOTE: Currently bypassing (possibly more than one) `TransformedDimension` wrappers to get
# the 'low', 'high' and 'base' attributes.
while isinstance(fidelity_dim, TransformedDimension):
fidelity_dim = fidelity_dim.original_dimension
assert isinstance(fidelity_dim, Fidelity)
self.min_budget = fidelity_dim.low
self.max_budget = fidelity_dim.high
self.eta = fidelity_dim.base
self._setup()
def _setup(self):
self.max_sh_iter = (
-int(np.log(self.min_budget / self.max_budget) / np.log(self.eta))
+ 1)
self.budgets = self.max_budget * np.power(
self.eta, -np.linspace(self.max_sh_iter - 1, 0, self.max_sh_iter))
self.bohb = CG_BOHB( # pylint: disable=attribute-defined-outside-init
configspace=convert_space(self.space),
min_points_in_model=self.min_points_in_model,
top_n_percent=self.top_n_percent,
num_samples=self.num_samples,
random_fraction=self.random_fraction,
bandwidth_factor=self.bandwidth_factor,
min_bandwidth=self.min_bandwidth,
)
self.bohb.configspace.seed(self.seed)
def _make_iteration(self, iteration):
ss = self.max_sh_iter - 1 - (iteration % self.max_sh_iter)
# number of configurations in that bracket
n0 = int(np.floor((self.max_sh_iter) / (ss + 1)) * self.eta**ss)
ns = [max(int(n0 * (self.eta**(-i))), 1) for i in range(ss + 1)]
return SuccessiveHalving(
HPB_iter=iteration,
num_configs=ns,
budgets=self.budgets[(-ss - 1):],
config_sampler=self.bohb.get_config,
)
def seed_rng(self, seed):
"""Seed the state of the random number generator.
Parameters
----------
seed: int
Integer seed for the random number generator.
"""
np.random.seed(seed)
if hasattr(self, "bohb"):
self.bohb.configspace.seed(seed)
@property
def state_dict(self):
"""Return a state dict that can be used to reset the state of the algorithm."""
state_dict = super().state_dict
state_dict["rng_state"] = np.random.get_state()
state_dict["eta"] = self.eta
state_dict["min_budget"] = self.min_budget
state_dict["max_budget"] = self.max_budget
state_dict["iteration"] = self.iteration
state_dict["iterations"] = copy.deepcopy(self.iterations)
state_dict["trial_meta"] = dict(self.trial_meta)
state_dict["trial_results"] = dict(self.trial_results)
state_dict["bohb"] = copy.deepcopy(self.bohb)
state_dict["strategy"] = self.strategy.state_dict
return state_dict
def set_state(self, state_dict):
"""Reset the state of the algorithm based on the given state_dict
:param state_dict: Dictionary representing state of an algorithm
"""
super().set_state(state_dict)
np.random.set_state(state_dict["rng_state"])
self.eta = state_dict["eta"]
self.min_budget = state_dict["min_budget"]
self.max_budget = state_dict["max_budget"]
self.iteration = state_dict["iteration"]
self.iterations = state_dict["iterations"]
self.trial_meta = state_dict["trial_meta"]
self.trial_results = state_dict["trial_results"]
self.bohb = state_dict["bohb"] # pylint: disable=attribute-defined-outside-init
self.strategy.set_state(state_dict["strategy"])
self._setup()
def suggest(self, num):
"""Suggest a number of new sets of parameters.
Parameters
----------
num: int, optional
Number of trials to suggest. The algorithm may return less than the number of trials
requested.
Returns
-------
list of trials or None
A list of trials representing values suggested by the algorithm. The algorithm may opt
out if it cannot make a good suggestion at the moment (it may be waiting for other
trials to complete), in which case it will return None.
Notes
-----
New parameters must be compliant with the problem's domain `orion.algo.space.Space`.
"""
def run_to_trial(run):
params = transform(run[1])
params[self.fidelity_index] = run[2]
return dict_to_trial(params, self.space)
def sample_iteration(iteration, trials):
while len(trials) < num and not iteration.is_finished:
run = iteration.get_next_run()
if run is None:
break
new_trial = run_to_trial(run)
# This means the job was already suggested and we have a result
result = self.trial_results.get(self.get_id(new_trial), None)
if result is not None:
job = FakeJob(run, new_trial)
job.result["loss"] = result
iteration.register_result(job)
self.bohb.new_result(job)
continue
self.trial_meta.setdefault(self.get_id(new_trial),
[]).append(run)
self.register(new_trial)
trials.append(new_trial)
trials = []
for it in self.iterations:
sample_iteration(it, trials)
# If we don't have enough trials and there are still
# some iterations left
if self.iteration < len(self.budgets):
self.iterations.append(self._make_iteration(self.iteration))
self.iteration += 1
sample_iteration(self.iterations[-1], trials)
return trials
def observe(self, trials):
"""Observe the `trials` new state of result.
Parameters
----------
trials: list of ``orion.core.worker.trial.Trial``
Trials from a `orion.algo.space.Space`.
"""
super().observe(trials)
for trial in trials:
if trial.status == "broken":
trial = self.strategy.infer(trial)
if trial.objective is not None:
self.trial_results[self.get_id(trial)] = trial.objective.value
runs = self.trial_meta.get(self.get_id(trial), [])
for run in runs:
job = FakeJob(run, trial)
self.iterations[job.id[0]].register_result(job)
self.bohb.new_result(job)
@property
def is_done(self):
"""Return True, if an algorithm holds that there can be no further improvement."""
return self.iteration == len(self.budgets) and all(
it.is_finished for it in self.iterations)
class TuneBOHB(SuggestionAlgorithm):
"""BOHB suggestion component.
Requires HpBandSter and ConfigSpace to be installed. You can install
HpBandSter and ConfigSpace with: ``pip install hpbandster ConfigSpace``.
This should be used in conjunction with HyperBandForBOHB.
Args:
space (ConfigurationSpace): Continuous ConfigSpace search space.
Parameters will be sampled from this space which will be used
to run trials.
bohb_config (dict): configuration for HpBandSter BOHB algorithm
max_concurrent (int): Number of maximum concurrent trials. Defaults
to 10.
metric (str): The training result objective value attribute.
mode (str): One of {min, max}. Determines whether objective is
minimizing or maximizing the metric attribute.
Example:
.. code-block:: python
import ConfigSpace as CS
config_space = CS.ConfigurationSpace()
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter('width', lower=0, upper=20))
config_space.add_hyperparameter(
CS.UniformFloatHyperparameter('height', lower=-100, upper=100))
config_space.add_hyperparameter(
CS.CategoricalHyperparameter(
name='activation', choices=['relu', 'tanh']))
algo = TuneBOHB(
config_space, max_concurrent=4, metric='mean_loss', mode='min')
bohb = HyperBandForBOHB(
time_attr='training_iteration',
metric='mean_loss',
mode='min',
max_t=100)
run(MyTrainableClass, scheduler=bohb, search_alg=algo)
"""
def __init__(self,
space,
bohb_config=None,
max_concurrent=10,
metric="neg_mean_loss",
mode="max"):
from hpbandster.optimizers.config_generators.bohb import BOHB
assert BOHB is not None, "HpBandSter must be installed!"
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!"
self._max_concurrent = max_concurrent
self.trial_to_params = {}
self.running = set()
self.paused = set()
self._metric = metric
if mode == "max":
self._metric_op = -1.
elif mode == "min":
self._metric_op = 1.
bohb_config = bohb_config or {}
self.bohber = BOHB(space, **bohb_config)
super(TuneBOHB, self).__init__(metric=self._metric, mode=mode)
def suggest(self, trial_id):
if len(self.running) < self._max_concurrent:
# This parameter is not used in hpbandster implementation.
config, info = self.bohber.get_config(None)
self.trial_to_params[trial_id] = copy.deepcopy(config)
self.running.add(trial_id)
return config
return None
def on_trial_result(self, trial_id, result):
if trial_id not in self.paused:
self.running.add(trial_id)
if "hyperband_info" not in result:
logger.warning("BOHB Info not detected in result. Are you using "
"HyperBandForBOHB as a scheduler?")
elif "budget" in result.get("hyperband_info", {}):
hbs_wrapper = self.to_wrapper(trial_id, result)
self.bohber.new_result(hbs_wrapper)
def on_trial_complete(self,
trial_id,
result=None,
error=False,
early_terminated=False):
del self.trial_to_params[trial_id]
if trial_id in self.paused:
self.paused.remove(trial_id)
if trial_id in self.running:
self.running.remove(trial_id)
def to_wrapper(self, trial_id, result):
return _BOHBJobWrapper(self._metric_op * result[self.metric],
result["hyperband_info"]["budget"],
self.trial_to_params[trial_id])
def on_pause(self, trial_id):
self.paused.add(trial_id)
self.running.remove(trial_id)
def on_unpause(self, trial_id):
self.paused.remove(trial_id)
self.running.add(trial_id)
