def _build_optimizer(self): """Set :attr:`optimizer` to the optimizing class used to both estimate the utility of sets of hyperparameters by learning from executed Experiments, and suggest points at which the objective should be evaluated""" self.optimizer = AskingOptimizer( dimensions=self.hyperparameter_space, base_estimator=self.base_estimator, n_initial_points=self.n_initial_points, acq_func=self.acquisition_function, acq_optimizer=self.acquisition_optimizer, random_state=self.random_state, acq_func_kwargs=self.acquisition_function_kwargs, acq_optimizer_kwargs=self.acquisition_optimizer_kwargs, )
class InformedOptimizationProtocol(BaseOptimizationProtocol, metaclass=ABCMeta): def __init__( self, target_metric=None, iterations=1, verbose=1, read_experiments=True, reporter_parameters=None, #################### Optimizer Class Parameters #################### base_estimator='GP', n_initial_points=10, acquisition_function='gp_hedge', acquisition_optimizer='auto', random_state=32, acquisition_function_kwargs=None, acquisition_optimizer_kwargs=None, #################### Minimizer Parameters #################### n_random_starts=10, callbacks=None, #################### Other Parameters #################### base_estimator_kwargs=None, ): """Base class for Informed Optimization Protocols Parameters ---------- target_metric: Tuple, default=('oof', <first key in :attr:`environment.Environment.metrics_map`>) A path denoting the metric to be used to compare completed Experiments within the Optimization Protocol. The first value should be one of ['oof', 'holdout', 'in_fold']. The second value should be the name of a metric being recorded according to the values supplied in :attr:`environment.Environment.metrics_params`. See the documentation for :func:`metrics.get_formatted_target_metric` for more info; any values returned by, or used as the `target_metric` input to this function are acceptable values for :attr:`BaseOptimizationProtocol.target_metric` iterations: Int, default=1 The number of distinct experiments to execute verbose: Int 0, 1, or 2, default=1 Verbosity mode for console logging. 0: Silent. 1: Show only logs from the Optimization Protocol. 2: In addition to logs shown when verbose=1, also show the logs from individual Experiments read_experiments: Boolean, default=True If True, all Experiment records that fit within the current :attr:`hyperparameter_space`, and are for the same :attr:`algorithm_name`, and match the current guidelines, will be read in and used to fit any optimizers reporter_parameters: Dict, or None, default=None Additional parameters passed to :meth:`reporting.OptimizationReporter.__init__` base_estimator: String in ['GP', 'GBRT', 'RF', 'ET', 'DUMMY'], or an `sklearn` regressor, default='GP' If one of the above strings, a default model of that type will be used. Else, should inherit from :class:`sklearn.base.RegressorMixin`, and its :meth:`predict` should have an optional `return_std` argument, which returns `std(Y | x)`, along with `E[Y | x]` n_initial_points: Int, default=10 The number of complete evaluation points necessary before allowing Experiments to be approximated with `base_estimator`. Any valid Experiment records found will count as initialization points. If enough Experiment records are not found, additional points will be randomly sampled acquisition_function: String in ['LCB', 'EI', 'PI', 'gp_hedge'], default='gp_hedge' Function to minimize over the posterior distribution. 'LCB': lower confidence bound. 'EI': negative expected improvement. 'PI': negative probability of improvement. 'gp_hedge': Probabilistically choose one of the preceding three acquisition functions at each iteration acquisition_optimizer: String in ['sampling', 'lbfgs', 'auto'], default='auto' Method to minimize the acquisition function. The fit model is updated with the optimal value obtained by optimizing `acquisition_function` with `acquisition_optimizer`. 'sampling': optimize by computing `acquisition_function` at `acquisition_optimizer_kwargs['n_points']` randomly sampled points. 'lbfgs': optimize by sampling `n_restarts_optimizer` random points, then run 'lbfgs' for 20 iterations with those points to find local minima, the optimal of which is used to update the prior. 'auto': configure on the basis of `base_estimator` and `dimensions` random_state: Int, `RandomState` instance, or None, default=None Set to something other than None for reproducible results acquisition_function_kwargs: Dict, or None, default=dict(xi=0.01, kappa=1.96) Additional arguments passed to the acquisition function acquisition_optimizer_kwargs: Dict, or None, default=dict(n_points=10000, n_restarts_optimizer=5, n_jobs=1) Additional arguments passed to the acquisition optimizer n_random_starts: Int, default=10 The number of Experiments to execute with random points before checking that `n_initial_points` have been evaluated callbacks: Callable, list of callables, or None, default=[] If callable, then `callbacks(self.optimizer_result)` is called after each update to :attr:`optimizer`. If list, then each callable is called base_estimator_kwargs: Dict, or None, default={} Additional arguments passed to `base_estimator` when it is initialized Notes ----- To provide initial input points for evaluation, individual Experiments can be executed prior to instantiating an Optimization Protocol. The results of these Experiments will automatically be detected and cherished by the optimizer. :class:`.InformedOptimizationProtocol` and its children in :mod:`.optimization` rely heavily on the utilities provided by the `Scikit-Optimize` library, so thank you to the creators and contributors for their excellent work.""" # TODO: Add 'EIps', and 'PIps' to the allowable `acquisition_function` values - Will need to return execution times #################### Optimizer Parameters #################### self.base_estimator = base_estimator self.n_initial_points = n_initial_points self.acquisition_function = acquisition_function self.acquisition_optimizer = acquisition_optimizer self.random_state = random_state self.acquisition_function_kwargs = dict(xi=0.01, kappa=1.96) self.acquisition_optimizer_kwargs = dict(n_points=10000, n_restarts_optimizer=5, n_jobs=1) self.acquisition_function_kwargs.update(acquisition_function_kwargs or {}) self.acquisition_optimizer_kwargs.update(acquisition_optimizer_kwargs or {}) #################### Minimizer Parameters #################### # TODO: n_random_starts does nothing currently - Fix that self.n_random_starts = n_random_starts self.callbacks = callbacks or [] #################### Other Parameters #################### self.base_estimator_kwargs = base_estimator_kwargs or {} #################### Placeholder Attributes #################### self.optimizer = None self.optimizer_result = None self.current_hyperparameters_list = None super().__init__(target_metric=target_metric, iterations=iterations, verbose=verbose, read_experiments=read_experiments, reporter_parameters=reporter_parameters) def _set_hyperparameter_space(self): """Initialize :attr:`hyperparameter_space` according to the provided hyperparameter search dimensions, and :attr:`base_estimator` and :attr:`optimizer`""" self.hyperparameter_space = Space(dimensions=self.dimensions) self._prepare_estimator() self._build_optimizer() def _prepare_estimator(self): """Initialize :attr:`base_estimator` with :attr:`hyperparameter_space` and any other kwargs, using `skopt.utils.cook_estimator`""" self.base_estimator = cook_estimator(self.base_estimator, space=self.hyperparameter_space, **self.base_estimator_kwargs) def _build_optimizer(self): """Set :attr:`optimizer` to the optimizing class used to both estimate the utility of sets of hyperparameters by learning from executed Experiments, and suggest points at which the objective should be evaluated""" self.optimizer = AskingOptimizer( dimensions=self.hyperparameter_space, base_estimator=self.base_estimator, n_initial_points=self.n_initial_points, acq_func=self.acquisition_function, acq_optimizer=self.acquisition_optimizer, random_state=self.random_state, acq_func_kwargs=self.acquisition_function_kwargs, acq_optimizer_kwargs=self.acquisition_optimizer_kwargs, ) def _execute_experiment(self): """After executing parent's :meth:`_execute_experiment`, fit :attr:`optimizer` with the set of hyperparameters that were used, and the utility of those hyperparameters""" super()._execute_experiment() # FLAG: Resolve switching between below options depending on `target_metric` # self.optimizer_result = self.optimizer.tell(self.current_hyperparameters_list, self.current_score, fit=True) self.optimizer_result = self.optimizer.tell( self.current_hyperparameters_list, -self.current_score, fit=True) # FLAG: Resolve switching between above options depending on `target_metric` if eval_callbacks(self.callbacks, self.optimizer_result): return def _get_current_hyperparameters(self): """Ask :attr:`optimizer` for the upcoming set of hyperparameters that should be searched, then format them to be used in the next Experiment Returns ------- current_hyperparameters: Dict The next set of hyperparameters that will be searched""" _current_hyperparameters = self.optimizer.ask() if _current_hyperparameters == self.current_hyperparameters_list: new_parameters = self.hyperparameter_space.rvs( random_state=None)[0] G.debug_('REPEATED asked={} new={}'.format( _current_hyperparameters, new_parameters)) _current_hyperparameters = new_parameters self.current_hyperparameters_list = _current_hyperparameters current_hyperparameters = dict( zip(self.hyperparameter_space.get_names(use_location=False), self.current_hyperparameters_list)) return current_hyperparameters def _find_similar_experiments(self): """After locating similar experiments by way of the parent's :meth:`_find_similar_experiments`, fit :attr:`optimizer` with the hyperparameters and results of each located experiment""" super()._find_similar_experiments() for _i, _experiment in enumerate(self.similar_experiments[::-1]): _hyperparameters = dimension_subset( _experiment[0], self.hyperparameter_space.get_names()) _evaluation = _experiment[1] _experiment_id = _experiment[2] if len(_experiment) > 2 else None self.logger.print_result(_hyperparameters, _evaluation, experiment_id=_experiment_id) # FLAG: Resolve switching between below options depending on `target_metric` # self.optimizer_result = self.optimizer.tell(_hyperparameters, _evaluation) self.optimizer_result = self.optimizer.tell( _hyperparameters, -_evaluation) # FLAG: Resolve switching between above options depending on `target_metric` # self.optimizer_result = self.optimizer.tell( # _hyperparameters, _evaluation, fit=(_i == len(self.similar_experiments) - 1)) if eval_callbacks(self.callbacks, self.optimizer_result): return self.optimizer_result # FLAG: Could wrap above `tell` call in try/except, then attempt `_tell` with improper dimensions def _validate_parameters(self): """Ensure provided input parameters are properly formatted""" super()._validate_parameters() #################### callbacks #################### self.callbacks = check_callback(self.callbacks) @property def search_space_size(self): """The number of different hyperparameter permutations possible given the current hyperparameter search dimensions. Returns ------- :attr:`_search_space_size`: Int, or `numpy.inf` Infinity will be returned if any of the following constraints are met: 1) the hyperparameter dimensions include any real-valued boundaries, 2) the boundaries include values that are neither categorical nor integer, or 3) the search space size is otherwise incalculable""" if self._search_space_size is None: self._search_space_size = len(self.hyperparameter_space) return self._search_space_size