def suggest(self, trial_id: str) -> Optional[Dict]: if not self._ax: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: return None if self._points_to_evaluate: config = self._points_to_evaluate.pop(0) parameters, trial_index = self._ax.attach_trial(config) else: try: parameters, trial_index = self._ax.get_next_trial() except (MaxParallelismReachedException, DataRequiredError): return None self._live_trial_mapping[trial_id] = trial_index return unflatten_dict(parameters)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._skopt_opt: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self._initial_points: suggested_config = self._initial_points.pop(0) skopt_config = [suggested_config[par] for par in self._parameters] else: skopt_config = self._skopt_opt.ask() suggested_config = dict(zip(self._parameters, skopt_config)) self._live_trial_mapping[trial_id] = skopt_config if self._convert_to_python: for k, v in list(suggested_config.items()): if isinstance(v, np.number): suggested_config[k] = v.item() return unflatten_dict(suggested_config)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format( cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format( cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if callable(self._space): # Define-by-run case if trial_id not in self._ot_trials: self._ot_trials[trial_id] = self._ot_study.ask() ot_trial = self._ot_trials[trial_id] params = self._suggest_from_define_by_run_func( self._space, ot_trial) else: # Use Optuna ask interface (since version 2.6.0) if trial_id not in self._ot_trials: self._ot_trials[trial_id] = self._ot_study.ask( fixed_distributions=self._space) ot_trial = self._ot_trials[trial_id] params = ot_trial.params return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format( cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format( cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if trial_id not in self._ot_trials: self._ot_trials[trial_id] = self._ot_study.ask() ot_trial = self._ot_trials[trial_id] # getattr will fetch the trial.suggest_ function on Optuna trials params = { args[0] if len(args) > 0 else kwargs["name"]: getattr( ot_trial, fn)(*args, **kwargs) for (fn, args, kwargs) in self._space } return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._opt: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if not self._live_trial_mapping: self._batch_filled = False if self._initial_points: params = self._initial_points.pop(0) suggestion = pd.DataFrame(params, index=[0]) else: if (self._batch_filled or len(self._live_trial_mapping) >= self._max_concurrent): return None if not self._suggestions_cache: suggestion = self._opt.suggest( n_suggestions=self._max_concurrent) self._suggestions_cache = suggestion.to_dict("records") params = self._suggestions_cache.pop(0) suggestion = pd.DataFrame(params, index=[0]) self._live_trial_mapping[trial_id] = suggestion if len(self._live_trial_mapping) >= self._max_concurrent: self._batch_filled = True return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]: """Return new point to be explored by black box function. Args: trial_id: Id of the trial. This is a short alphanumerical string. Returns: Either a dictionary describing the new point to explore or None, when no new point is to be explored for the time being. """ if not self.optimizer: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self._points_to_evaluate: config = self._points_to_evaluate.pop(0) else: # We compute the new point to explore config = self.optimizer.suggest(self.utility) config_hash = _dict_hash(config, self.repeat_float_precision) # Check if already computed already_seen = config_hash in self._config_counter self._config_counter[config_hash] += 1 top_repeats = max(self._config_counter.values()) # If patience is set and we've repeated a trial numerous times, # we terminate the experiment. if self._patience is not None and top_repeats > self._patience: return Searcher.FINISHED # If we have seen a value before, we'll skip it. if already_seen and self._skip_duplicate: logger.info("Skipping duplicated config: {}.".format(config)) return None # If we are still in the random search part and we are waiting for # trials to complete if len(self._buffered_trial_results) < self.random_search_trials: # We check if we have already maxed out the number of requested # random search trials if self._total_random_search_trials == self.random_search_trials: # If so we stop the suggestion and return None return None # Otherwise we increase the total number of rndom search trials if config: self._total_random_search_trials += 1 # Save the new trial to the trial mapping self._live_trial_mapping[trial_id] = config # Return a deep copy of the mapping return unflatten_dict(config)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._nevergrad_opt: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: return None if self._points_to_evaluate is not None: if len(self._points_to_evaluate) > 0: point_to_evaluate = self._points_to_evaluate.pop(0) self._nevergrad_opt.suggest(point_to_evaluate) suggested_config = self._nevergrad_opt.ask() self._live_trial_mapping[trial_id] = suggested_config # in v0.2.0+, output of ask() is a Candidate, # with fields args and kwargs if not suggested_config.kwargs: if self._parameters: return unflatten_dict( dict(zip(self._parameters, suggested_config.args[0]))) return unflatten_dict(suggested_config.value) else: return unflatten_dict(suggested_config.kwargs)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._opt: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self._initial_points: suggested_config = self._initial_points[0] del self._initial_points[0] else: try: suggested_config = self._opt.ask() except Exception as exc: logger.warning( "Dragonfly errored when querying. This may be due to a " "higher level of parallelism than supported. Try reducing " "parallelism in the experiment: %s", str(exc), ) return None self._live_trial_mapping[trial_id] = suggested_config config = dict(zip(self._point_parameter_names, suggested_config)) # Keep backwards compatibility config.update(point=suggested_config) return unflatten_dict(config)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._dim_dict or not self.optimizer: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="dim_dict")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) _solution = self.optimizer.suggest() if _solution == "FINISHED": if ray.__version__ >= "0.8.7": return Searcher.FINISHED else: return None if _solution: self.solution_dict[str(trial_id)] = _solution _x = _solution.get_x() new_trial = dict(zip(self._dim_keys, _x)) self._live_trial_mapping[trial_id] = new_trial return unflatten_dict(new_trial)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if isinstance(self._space, list): # Keep for backwards compatibility # Deprecate: 1.5 if trial_id not in self._ot_trials: self._ot_trials[trial_id] = self._ot_study.ask() ot_trial = self._ot_trials[trial_id] # getattr will fetch the trial.suggest_ function on Optuna trials params = { args[0] if len(args) > 0 else kwargs["name"]: getattr(ot_trial, fn)(*args, **kwargs) for (fn, args, kwargs) in self._space } else: # Use Optuna ask interface (since version 2.6.0) if trial_id not in self._ot_trials: self._ot_trials[trial_id] = self._ot_study.ask( fixed_distributions=self._space) ot_trial = self._ot_trials[trial_id] params = ot_trial.params return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if trial_id not in self._ot_trials: ot_trial_id = self._storage.create_new_trial( self._ot_study._study_id) self._ot_trials[trial_id] = ot.trial.Trial(self._ot_study, ot_trial_id) ot_trial = self._ot_trials[trial_id] if self._points_to_evaluate: params = self._points_to_evaluate.pop(0) else: # getattr will fetch the trial.suggest_ function on Optuna trials params = { args[0] if len(args) > 0 else kwargs["name"]: getattr(ot_trial, fn)(*args, **kwargs) for (fn, args, kwargs) in self._space } return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self.domain: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format( cls=self.__class__.__name__, space="space" ) ) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format( cls=self.__class__.__name__, metric=self._metric, mode=self._mode ) ) if self._points_to_evaluate > 0: new_trial = self._hpopt_trials.trials[self._points_to_evaluate - 1] self._points_to_evaluate -= 1 else: new_ids = self._hpopt_trials.new_trial_ids(1) self._hpopt_trials.refresh() # Get new suggestion from Hyperopt new_trials = self.algo( new_ids, self.domain, self._hpopt_trials, self.rstate.randint(2 ** 31 - 1), ) self._hpopt_trials.insert_trial_docs(new_trials) self._hpopt_trials.refresh() new_trial = new_trials[0] self._live_trial_mapping[trial_id] = (new_trial["tid"], new_trial) # Taken from HyperOpt.base.evaluate config = hpo.base.spec_from_misc(new_trial["misc"]) # We have to flatten nested spaces here so parameter names match config = flatten_dict(config, flatten_list=True) ctrl = hpo.base.Ctrl(self._hpopt_trials, current_trial=new_trial) memo = self.domain.memo_from_config(config) hpo.utils.use_obj_for_literal_in_memo( self.domain.expr, ctrl, hpo.base.Ctrl, memo ) suggested_config = hpo.pyll.rec_eval( self.domain.expr, memo=memo, print_node_on_error=self.domain.rec_eval_print_node_on_error, ) return copy.deepcopy(suggested_config)
def add_evaluated_point( self, parameters: Dict, value: float, error: bool = False, pruned: bool = False, intermediate_values: Optional[List[float]] = None, ): if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format( cls=self.__class__.__name__, space="space" ) ) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format( cls=self.__class__.__name__, metric=self._metric, mode=self._mode ) ) if callable(self._space): raise TypeError( "Define-by-run function passed in `space` argument is not " "yet supported when using `evaluated_rewards`. Please provide " "an `OptunaDistribution` dict or pass a Ray Tune " "search space to `tune.run()`." ) ot_trial_state = OptunaTrialState.COMPLETE if error: ot_trial_state = OptunaTrialState.FAIL elif pruned: ot_trial_state = OptunaTrialState.PRUNED if intermediate_values: intermediate_values_dict = { i: value for i, value in enumerate(intermediate_values) } else: intermediate_values_dict = None trial = ot.trial.create_trial( state=ot_trial_state, value=value, params=parameters, distributions=self._space, intermediate_values=intermediate_values_dict, ) self._ot_study.add_trial(trial)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._ax: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: return None parameters, trial_index = self._ax.get_next_trial() self._live_trial_mapping[trial_id] = trial_index return unflatten_dict(parameters)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if len(self.running) < self._max_concurrent: # This parameter is not used in hpbandster implementation. config, info = self.bohber.get_config(None) self.trial_to_params[trial_id] = copy.deepcopy(config) self.running.add(trial_id) return unflatten_dict(config) return None
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self._points_to_evaluate: config = self._points_to_evaluate.pop(0) else: # This parameter is not used in hpbandster implementation. config, _ = self.bohber.get_config(None) self.trial_to_params[trial_id] = copy.deepcopy(config) return unflatten_list_dict(config)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._opt: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self._initial_points: params = self._initial_points.pop(0) suggestion = pd.DataFrame(params, index=[0]) else: suggestion = self._opt.suggest() params = suggestion.iloc[0].to_dict() self._live_trial_mapping[trial_id] = suggestion return unflatten_dict(params)
def suggest(self, trial_id: str) -> Optional[Dict]: if not self._skopt_opt: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: return None if self._initial_points: suggested_config = self._initial_points[0] del self._initial_points[0] else: suggested_config = self._skopt_opt.ask() self._live_trial_mapping[trial_id] = suggested_config return unflatten_dict(dict(zip(self._parameters, suggested_config)))
def add_evaluated_point(self, parameters: Dict, value: float, error: bool = False, pruned: bool = False, intermediate_values: Optional[List[float]] = None): if not self._space: raise RuntimeError( UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space")) if not self._metric or not self._mode: raise RuntimeError( UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)) ot_trial_state = OptunaTrialState.COMPLETE if error: ot_trial_state = OptunaTrialState.FAIL elif pruned: ot_trial_state = OptunaTrialState.PRUNED if intermediate_values: intermediate_values_dict = { i: value for i, value in enumerate(intermediate_values) } else: intermediate_values_dict = None trial = ot.trial.create_trial( state=ot_trial_state, value=value, params=parameters, distributions=self._space, intermediate_values=intermediate_values_dict) self._ot_study.add_trial(trial)