def my_suggest(new_ids,
domain,
trials,
seed,
prior_weight=_default_prior_weight,
n_startup_jobs=_default_n_startup_jobs,
n_EI_candidates=_default_n_EI_candidates,
gamma=_default_gamma):
return tpe.suggest(new_ids, domain, trials, seed, prior_weight,
n_startup_jobs, n_EI_candidates, gamma)
def _hyperopt_suggest(self):
new_ids = self.trials.new_trial_ids(1)
assert len(new_ids) == 1
self.trials.refresh()
seed = random_seed(self.random)
new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed)
assert len(new_trials) == 1
self.trials.insert_trial_docs(new_trials)
self.trials.refresh()
new_trial, = new_trials # extract singleton
return new_trial
def _suggest(self):
"""Helper function to `suggest` that does the work of calling
`hyperopt` via its dumb API.
"""
new_ids = self.trials.new_trial_ids(1)
assert len(new_ids) == 1
self.trials.refresh()
seed = random_seed(self.random)
new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed)
assert len(new_trials) == 1
self.trials.insert_trial_docs(new_trials)
self.trials.refresh()
new_trial, = new_trials # extract singleton
return new_trial
def _internal_ask_candidate(self) -> p.Parameter:
# Inspired from FMinIter class (hyperopt)
next_id = self.trials.new_trial_ids(1)
new_trial = tpe.suggest(next_id, self.domain, self.trials,
self._rng.randint(2**31 - 1),
**self.tpe_args)[0]
self.trials.insert_trial_doc(new_trial)
self.trials.refresh()
candidate = self.parametrization.spawn_child()
if self._transform:
data = np.array([
new_trial["misc"]["vals"][f"x_{i}"][0]
for i in range(self.dimension)
])
candidate = candidate.set_standardized_data(
self._transform.backward(data))
# For consistency, we need to update hyperopt history
# when standardized data is changed
if any(data != self._transform.forward(
candidate.get_standardized_data(
reference=self.parametrization))):
for it, val in enumerate(
self._transform.forward(
candidate.get_standardized_data(
reference=self.parametrization))):
self.trials._dynamic_trials[
next_id[0]]["misc"]["vals"][f"x_{it}"][0] = val
else:
spec = hyperopt.base.spec_from_misc(new_trial["misc"])
config = hyperopt.space_eval(self.space, spec)
candidate.value = _hp_dict_to_parametrization(config)
candidate._meta["trial_id"] = new_trial["tid"]
return candidate
def suggest(*args, **kwargs):
return tpe.suggest(*args,
**kwargs,
gamma=self.gamma,
n_startup_jobs=self.seeds)
def _fit_on_prepared_data(self, metadata):
trials = Trials()
domain = Domain(None, self.hyperopt_search_space)
rand = np.random.RandomState(constants.RANDOM_SEED)
all_estimators = []
all_estimator_results = {}
for i in range(0, self.num_params, self.parallelism):
n = min(self.num_params - i, self.parallelism)
# Using HyperOpt TPE to generate parameters
hyperopt_params = []
for j in range(i, i + n):
new_param = tpe.suggest([j], domain, trials, rand.randint(0, 2 ** 31 - 1))
new_param[0]['status'] = STATUS_RUNNING
trials.insert_trial_docs(new_param)
trials.refresh()
hyperopt_params.append(new_param[0])
# Generating Cerebro params from HyperOpt params
estimator_param_maps = []
for hyperopt_param in hyperopt_params:
param = {}
for k in hyperopt_param['misc']['vals']:
val = hyperopt_param['misc']['vals'][k][0].item()
if isinstance(self.search_space[k], _HPChoice):
# if the hyperparamer is a choice the index is returned
val = self.search_space[k].options[val]
param[k] = val
estimator_param_maps.append(param)
# Generating Cerebro estimators
estimators = [self._estimator_gen_fn_wrapper(param) for param in estimator_param_maps]
estimator_results = {model.getRunId(): {} for model in estimators}
# log hyperparameters to TensorBoard
self._log_hp_to_tensorboard(estimators, estimator_param_maps)
# Trains the models up to the number of epochs specified. For each iteration also performs validation
for epoch in range(self.num_epochs):
epoch_results = self.backend.train_for_one_epoch(estimators, self.store, self.feature_cols,
self.label_cols)
update_model_results(estimator_results, epoch_results)
epoch_results = self.backend.train_for_one_epoch(estimators, self.store, self.feature_cols,
self.label_cols, is_train=False)
update_model_results(estimator_results, epoch_results)
self._log_epoch_metrics_to_tensorboard(estimators, estimator_results)
all_estimators.extend(estimators)
all_estimator_results.update(estimator_results)
# HyperOpt TPE update
for i, hyperopt_param in enumerate(hyperopt_params):
hyperopt_param['status'] = STATUS_OK
hyperopt_param['result'] = {'loss': estimator_results[estimators[i].getRunId()][
'val_' + self.evaluation_metric][-1], 'status': STATUS_OK}
trials.refresh()
# find the best model and crate ModelSearchModel
models = [est.create_model(all_estimator_results[est.getRunId()], est.getRunId(), metadata) for est in
all_estimators]
val_metrics = [all_estimator_results[est.getRunId()]['val_' + self.evaluation_metric][-1] for est in
all_estimators]
best_model = models[np.argmin(val_metrics)]
return ModelSelectionResult(best_model, estimator_results, models, [x+"__output" for x in self.label_cols])
def tpeSuggestWrapper(*args, **kwargs):
modifiedKwargs = dict(kwargs)
modifiedKwargs["seed"] = options.seed
return tpe.suggest(*args, **modifiedKwargs)
def tpeSuggestWrapper(*args, **kwargs):
modifiedKwargs = dict(kwargs)
modifiedKwargs["seed"] = options.seed
return tpe.suggest(*args, **modifiedKwargs)
def search(self, run_name, store, context, hp_records, runs):
def make_trial(tid, arg_dict, loss_value):
trial = {"book_time": None, "exp_key": None, "owner": None, "refresh_time": None, "spec": None, "state": 0, "tid": tid, "version": 0}
#trial["result"] = {"status": "New"}
misc = {}
trial["misc"] = misc
misc["cmd"] = ("domain_attachment", "FMinIter_Domain")
misc["idxs"] = {key: [tid] for key in arg_dict.keys()}
misc["tid"] = tid
misc["vals"] = arg_dict
trial["state"] = 2 # done
trial["result"] = {"loss": loss_value, "status": "ok"}
#trial["refresh_time"] = coarse_utcnow()
return trial
dummy_loss = lambda x: None
param_space = {r["name"]: r["space_func"] for r in hp_records}
domain = base.Domain(dummy_loss, param_space)
rstate = np.random.RandomState()
# convert runs to Trials
trial_list = []
for run in runs:
# don't trip over inappropriate runs
if (not "run_name" in run) or (not "hparams" in run) or (not context.primary_metric in run):
continue
run_name = run["run_name"]
arg_dict = run["hparams"]
loss_value = run[context.primary_metric]
if context.maximize_metric:
loss_value = -loss_value
# extract a unique int from run_name (parent.childnum)
tid = run_helper.get_int_from_run_name(run_name)
trial = make_trial(tid, arg_dict, loss_value)
trial_list.append(trial)
# finally, add our trial_list to trials
trials = Trials()
trials.insert_trial_docs(trial_list)
trials.refresh()
# get next suggested hyperparameter values from TPE algorithm
tid = run_helper.get_int_from_run_name(run_name)
min_trials = 3 # before this, just do rand sampling
seed = rstate.randint(2 ** 31 - 1)
if len(trials) < min_trials:
new_trials = rand.suggest([tid], domain, trials, seed)
else:
new_trials = tpe.suggest([tid], domain, trials, seed)
# apply the suggested hparam values
trial = new_trials[0]
arg_dict = trial["misc"]["vals"]
arg_dict = self.fixup_hyperopt_hparams(param_space, arg_dict)
return arg_dict
def suggest_model(sid, trials):
sug = tpe.suggest((sid, ), domain, trials, rstate.randint(2**31 - 1))[0]
suggestions[sid] = sug
return sug