def my_suggest(new_ids, domain, trials, seed, prior_weight=_default_prior_weight, n_startup_jobs=_default_n_startup_jobs, n_EI_candidates=_default_n_EI_candidates, gamma=_default_gamma): return tpe.suggest(new_ids, domain, trials, seed, prior_weight, n_startup_jobs, n_EI_candidates, gamma)
def _hyperopt_suggest(self): new_ids = self.trials.new_trial_ids(1) assert len(new_ids) == 1 self.trials.refresh() seed = random_seed(self.random) new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed) assert len(new_trials) == 1 self.trials.insert_trial_docs(new_trials) self.trials.refresh() new_trial, = new_trials # extract singleton return new_trial
def _suggest(self): """Helper function to `suggest` that does the work of calling `hyperopt` via its dumb API. """ new_ids = self.trials.new_trial_ids(1) assert len(new_ids) == 1 self.trials.refresh() seed = random_seed(self.random) new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed) assert len(new_trials) == 1 self.trials.insert_trial_docs(new_trials) self.trials.refresh() new_trial, = new_trials # extract singleton return new_trial
def _internal_ask_candidate(self) -> p.Parameter: # Inspired from FMinIter class (hyperopt) next_id = self.trials.new_trial_ids(1) new_trial = tpe.suggest(next_id, self.domain, self.trials, self._rng.randint(2**31 - 1), **self.tpe_args)[0] self.trials.insert_trial_doc(new_trial) self.trials.refresh() candidate = self.parametrization.spawn_child() if self._transform: data = np.array([ new_trial["misc"]["vals"][f"x_{i}"][0] for i in range(self.dimension) ]) candidate = candidate.set_standardized_data( self._transform.backward(data)) # For consistency, we need to update hyperopt history # when standardized data is changed if any(data != self._transform.forward( candidate.get_standardized_data( reference=self.parametrization))): for it, val in enumerate( self._transform.forward( candidate.get_standardized_data( reference=self.parametrization))): self.trials._dynamic_trials[ next_id[0]]["misc"]["vals"][f"x_{it}"][0] = val else: spec = hyperopt.base.spec_from_misc(new_trial["misc"]) config = hyperopt.space_eval(self.space, spec) candidate.value = _hp_dict_to_parametrization(config) candidate._meta["trial_id"] = new_trial["tid"] return candidate
def suggest(*args, **kwargs): return tpe.suggest(*args, **kwargs, gamma=self.gamma, n_startup_jobs=self.seeds)
def _fit_on_prepared_data(self, metadata): trials = Trials() domain = Domain(None, self.hyperopt_search_space) rand = np.random.RandomState(constants.RANDOM_SEED) all_estimators = [] all_estimator_results = {} for i in range(0, self.num_params, self.parallelism): n = min(self.num_params - i, self.parallelism) # Using HyperOpt TPE to generate parameters hyperopt_params = [] for j in range(i, i + n): new_param = tpe.suggest([j], domain, trials, rand.randint(0, 2 ** 31 - 1)) new_param[0]['status'] = STATUS_RUNNING trials.insert_trial_docs(new_param) trials.refresh() hyperopt_params.append(new_param[0]) # Generating Cerebro params from HyperOpt params estimator_param_maps = [] for hyperopt_param in hyperopt_params: param = {} for k in hyperopt_param['misc']['vals']: val = hyperopt_param['misc']['vals'][k][0].item() if isinstance(self.search_space[k], _HPChoice): # if the hyperparamer is a choice the index is returned val = self.search_space[k].options[val] param[k] = val estimator_param_maps.append(param) # Generating Cerebro estimators estimators = [self._estimator_gen_fn_wrapper(param) for param in estimator_param_maps] estimator_results = {model.getRunId(): {} for model in estimators} # log hyperparameters to TensorBoard self._log_hp_to_tensorboard(estimators, estimator_param_maps) # Trains the models up to the number of epochs specified. For each iteration also performs validation for epoch in range(self.num_epochs): epoch_results = self.backend.train_for_one_epoch(estimators, self.store, self.feature_cols, self.label_cols) update_model_results(estimator_results, epoch_results) epoch_results = self.backend.train_for_one_epoch(estimators, self.store, self.feature_cols, self.label_cols, is_train=False) update_model_results(estimator_results, epoch_results) self._log_epoch_metrics_to_tensorboard(estimators, estimator_results) all_estimators.extend(estimators) all_estimator_results.update(estimator_results) # HyperOpt TPE update for i, hyperopt_param in enumerate(hyperopt_params): hyperopt_param['status'] = STATUS_OK hyperopt_param['result'] = {'loss': estimator_results[estimators[i].getRunId()][ 'val_' + self.evaluation_metric][-1], 'status': STATUS_OK} trials.refresh() # find the best model and crate ModelSearchModel models = [est.create_model(all_estimator_results[est.getRunId()], est.getRunId(), metadata) for est in all_estimators] val_metrics = [all_estimator_results[est.getRunId()]['val_' + self.evaluation_metric][-1] for est in all_estimators] best_model = models[np.argmin(val_metrics)] return ModelSelectionResult(best_model, estimator_results, models, [x+"__output" for x in self.label_cols])
def tpeSuggestWrapper(*args, **kwargs): modifiedKwargs = dict(kwargs) modifiedKwargs["seed"] = options.seed return tpe.suggest(*args, **modifiedKwargs)
def tpeSuggestWrapper(*args, **kwargs): modifiedKwargs = dict(kwargs) modifiedKwargs["seed"] = options.seed return tpe.suggest(*args, **modifiedKwargs)
def search(self, run_name, store, context, hp_records, runs): def make_trial(tid, arg_dict, loss_value): trial = {"book_time": None, "exp_key": None, "owner": None, "refresh_time": None, "spec": None, "state": 0, "tid": tid, "version": 0} #trial["result"] = {"status": "New"} misc = {} trial["misc"] = misc misc["cmd"] = ("domain_attachment", "FMinIter_Domain") misc["idxs"] = {key: [tid] for key in arg_dict.keys()} misc["tid"] = tid misc["vals"] = arg_dict trial["state"] = 2 # done trial["result"] = {"loss": loss_value, "status": "ok"} #trial["refresh_time"] = coarse_utcnow() return trial dummy_loss = lambda x: None param_space = {r["name"]: r["space_func"] for r in hp_records} domain = base.Domain(dummy_loss, param_space) rstate = np.random.RandomState() # convert runs to Trials trial_list = [] for run in runs: # don't trip over inappropriate runs if (not "run_name" in run) or (not "hparams" in run) or (not context.primary_metric in run): continue run_name = run["run_name"] arg_dict = run["hparams"] loss_value = run[context.primary_metric] if context.maximize_metric: loss_value = -loss_value # extract a unique int from run_name (parent.childnum) tid = run_helper.get_int_from_run_name(run_name) trial = make_trial(tid, arg_dict, loss_value) trial_list.append(trial) # finally, add our trial_list to trials trials = Trials() trials.insert_trial_docs(trial_list) trials.refresh() # get next suggested hyperparameter values from TPE algorithm tid = run_helper.get_int_from_run_name(run_name) min_trials = 3 # before this, just do rand sampling seed = rstate.randint(2 ** 31 - 1) if len(trials) < min_trials: new_trials = rand.suggest([tid], domain, trials, seed) else: new_trials = tpe.suggest([tid], domain, trials, seed) # apply the suggested hparam values trial = new_trials[0] arg_dict = trial["misc"]["vals"] arg_dict = self.fixup_hyperopt_hparams(param_space, arg_dict) return arg_dict
def suggest_model(sid, trials): sug = tpe.suggest((sid, ), domain, trials, rstate.randint(2**31 - 1))[0] suggestions[sid] = sug return sug