def suggest(self, trial_id: str) -> Optional[Dict]:
"""Return new point to be explored by black box function.
Args:
trial_id: Id of the trial.
This is a short alphanumerical string.
Returns:
Either a dictionary describing the new point to explore or
None, when no new point is to be explored for the time being.
"""
if not self.optimizer:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self._points_to_evaluate:
config = self._points_to_evaluate.pop(0)
else:
# We compute the new point to explore
config = self.optimizer.suggest(self.utility)
config_hash = _dict_hash(config, self.repeat_float_precision)
# Check if already computed
already_seen = config_hash in self._config_counter
self._config_counter[config_hash] += 1
top_repeats = max(self._config_counter.values())
# If patience is set and we've repeated a trial numerous times,
# we terminate the experiment.
if self._patience is not None and top_repeats > self._patience:
return Searcher.FINISHED
# If we have seen a value before, we'll skip it.
if already_seen and self._skip_duplicate:
logger.info("Skipping duplicated config: {}.".format(config))
return None
# If we are still in the random search part and we are waiting for
# trials to complete
if len(self._buffered_trial_results) < self.random_search_trials:
# We check if we have already maxed out the number of requested
# random search trials
if self._total_random_search_trials == self.random_search_trials:
# If so we stop the suggestion and return None
return None
# Otherwise we increase the total number of rndom search trials
if config:
self._total_random_search_trials += 1
# Save the new trial to the trial mapping
self._live_trial_mapping[trial_id] = config
# Return a deep copy of the mapping
return unflatten_dict(config)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._space:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if callable(self._space):
# Define-by-run case
if trial_id not in self._ot_trials:
self._ot_trials[trial_id] = self._ot_study.ask()
ot_trial = self._ot_trials[trial_id]
params = self._suggest_from_define_by_run_func(
self._space, ot_trial)
else:
# Use Optuna ask interface (since version 2.6.0)
if trial_id not in self._ot_trials:
self._ot_trials[trial_id] = self._ot_study.ask(
fixed_distributions=self._space)
ot_trial = self._ot_trials[trial_id]
params = ot_trial.params
return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._dim_dict or not self.optimizer:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="dim_dict"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
_solution = self.optimizer.suggest()
if _solution == "FINISHED":
if ray.__version__ >= "0.8.7":
return Searcher.FINISHED
else:
return None
if _solution:
self.solution_dict[str(trial_id)] = _solution
_x = _solution.get_x()
new_trial = dict(zip(self._dim_keys, _x))
self._live_trial_mapping[trial_id] = new_trial
return unflatten_dict(new_trial)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._space:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
max_concurrent = (self._max_concurrent
if self._max_concurrent > 0 else float("inf"))
if len(self.running) >= max_concurrent:
return None
if self._points_to_evaluate:
config = self._points_to_evaluate.pop(0)
else:
# This parameter is not used in hpbandster implementation.
config, _ = self.bohber.get_config(None)
self.trial_to_params[trial_id] = copy.deepcopy(config)
self.running.add(trial_id)
return unflatten_list_dict(config)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._opt:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if not self._live_trial_mapping:
self._batch_filled = False
if self._initial_points:
params = self._initial_points.pop(0)
suggestion = pd.DataFrame([params], index=[0])
else:
if (self._batch_filled
or len(self._live_trial_mapping) >= self._max_concurrent):
return None
if not self._suggestions_cache:
suggestion = self._opt.suggest(
n_suggestions=self._max_concurrent)
self._suggestions_cache = suggestion.to_dict("records")
params = self._suggestions_cache.pop(0)
suggestion = pd.DataFrame([params], index=[0])
self._live_trial_mapping[trial_id] = suggestion
if len(self._live_trial_mapping) >= self._max_concurrent:
self._batch_filled = True
return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._skopt_opt:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self._initial_points:
suggested_config = self._initial_points.pop(0)
skopt_config = [suggested_config[par] for par in self._parameters]
else:
skopt_config = self._skopt_opt.ask()
suggested_config = dict(zip(self._parameters, skopt_config))
self._live_trial_mapping[trial_id] = skopt_config
if self._convert_to_python:
for k, v in list(suggested_config.items()):
if isinstance(v, np.number):
suggested_config[k] = v.item()
return unflatten_dict(suggested_config)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._nevergrad_opt:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"
)
)
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__, metric=self._metric, mode=self._mode
)
)
suggested_config = self._nevergrad_opt.ask()
self._live_trial_mapping[trial_id] = suggested_config
# in v0.2.0+, output of ask() is a Candidate,
# with fields args and kwargs
if not suggested_config.kwargs:
if self._parameters:
return unflatten_dict(
dict(zip(self._parameters, suggested_config.args[0]))
)
return unflatten_dict(suggested_config.value)
else:
return unflatten_dict(suggested_config.kwargs)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._opt:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self._initial_points:
suggested_config = self._initial_points[0]
del self._initial_points[0]
else:
try:
suggested_config = self._opt.ask()
except Exception as exc:
logger.warning(
"Dragonfly errored when querying. This may be due to a "
"higher level of parallelism than supported. Try reducing "
"parallelism in the experiment: %s",
str(exc),
)
return None
self._live_trial_mapping[trial_id] = suggested_config
config = dict(zip(self._point_parameter_names, suggested_config))
# Keep backwards compatibility
config.update(point=suggested_config)
return unflatten_dict(config)
def add_evaluated_point(
self,
parameters: Dict,
value: float,
error: bool = False,
pruned: bool = False,
intermediate_values: Optional[List[float]] = None,
):
if not self._space:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if callable(self._space):
raise TypeError(
"Define-by-run function passed in `space` argument is not "
"yet supported when using `evaluated_rewards`. Please provide "
"an `OptunaDistribution` dict or pass a Ray Tune "
"search space to `tune.run()`.")
ot_trial_state = OptunaTrialState.COMPLETE
if error:
ot_trial_state = OptunaTrialState.FAIL
elif pruned:
ot_trial_state = OptunaTrialState.PRUNED
if intermediate_values:
intermediate_values_dict = {
i: value
for i, value in enumerate(intermediate_values)
}
else:
intermediate_values_dict = None
trial = ot.trial.create_trial(
state=ot_trial_state,
value=value,
params=parameters,
distributions=self._space,
intermediate_values=intermediate_values_dict,
)
self._ot_study.add_trial(trial)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self.domain:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self._points_to_evaluate > 0:
new_trial = self._hpopt_trials.trials[self._points_to_evaluate - 1]
self._points_to_evaluate -= 1
else:
new_ids = self._hpopt_trials.new_trial_ids(1)
self._hpopt_trials.refresh()
# Get new suggestion from Hyperopt
new_trials = self.algo(
new_ids,
self.domain,
self._hpopt_trials,
self.rstate.randint(2**31 - 1),
)
self._hpopt_trials.insert_trial_docs(new_trials)
self._hpopt_trials.refresh()
new_trial = new_trials[0]
self._live_trial_mapping[trial_id] = (new_trial["tid"], new_trial)
# Taken from HyperOpt.base.evaluate
config = hpo.base.spec_from_misc(new_trial["misc"])
# We have to flatten nested spaces here so parameter names match
config = flatten_dict(config, flatten_list=True)
ctrl = hpo.base.Ctrl(self._hpopt_trials, current_trial=new_trial)
memo = self.domain.memo_from_config(config)
hpo.utils.use_obj_for_literal_in_memo(self.domain.expr, ctrl,
hpo.base.Ctrl, memo)
suggested_config = hpo.pyll.rec_eval(
self.domain.expr,
memo=memo,
print_node_on_error=self.domain.rec_eval_print_node_on_error,
)
return copy.deepcopy(suggested_config)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._ax:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__,
space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self._points_to_evaluate:
config = self._points_to_evaluate.pop(0)
parameters, trial_index = self._ax.attach_trial(config)
else:
try:
parameters, trial_index = self._ax.get_next_trial()
except (MaxParallelismReachedException, DataRequiredError):
return None
self._live_trial_mapping[trial_id] = trial_index
return unflatten_dict(parameters)
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self.domain:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"
)
)
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__, metric=self._metric, mode=self._mode
)
)
if self._points_to_evaluate > 0:
using_point_to_evaluate = True
new_trial = self._hpopt_trials.trials[self._points_to_evaluate - 1]
self._points_to_evaluate -= 1
else:
using_point_to_evaluate = False
new_ids = self._hpopt_trials.new_trial_ids(1)
self._hpopt_trials.refresh()
# Get new suggestion from Hyperopt
new_trials = self.algo(
new_ids,
self.domain,
self._hpopt_trials,
self.rstate.randint(2 ** 31 - 1),
)
self._hpopt_trials.insert_trial_docs(new_trials)
self._hpopt_trials.refresh()
new_trial = new_trials[0]
self._live_trial_mapping[trial_id] = (new_trial["tid"], new_trial)
# Taken from HyperOpt.base.evaluate
config = hpo.base.spec_from_misc(new_trial["misc"])
# We have to flatten nested spaces here so parameter names match
config = flatten_dict(config, flatten_list=True)
ctrl = hpo.base.Ctrl(self._hpopt_trials, current_trial=new_trial)
memo = self.domain.memo_from_config(config)
hpo.utils.use_obj_for_literal_in_memo(
self.domain.expr, ctrl, hpo.base.Ctrl, memo
)
try:
suggested_config = hpo.pyll.rec_eval(
self.domain.expr,
memo=memo,
print_node_on_error=self.domain.rec_eval_print_node_on_error,
)
except (AssertionError, TypeError) as e:
if using_point_to_evaluate and (
isinstance(e, AssertionError) or "GarbageCollected" in str(e)
):
raise ValueError(
"HyperOpt encountered a GarbageCollected switch argument. "
"Usually this is caused by a config in "
"`points_to_evaluate` "
"missing a key present in `space`. Ensure that "
"`points_to_evaluate` contains "
"all non-constant keys from `space`.\n"
"Config from `points_to_evaluate`: "
f"{config}\n"
"HyperOpt search space: "
f"{self._space}"
) from e
raise e
return copy.deepcopy(suggested_config)