def evaluate(self, study: Study, params: Optional[List[str]]) -> Dict[str, float]: distributions = _get_distributions(study, params) params_data, values_data = _get_study_data(study, distributions) evaluator = fANOVA( X=params_data, Y=values_data, config_space=_get_configuration_space(distributions), max_features=max(1, int(params_data.shape[1] * 0.7)), ) individual_importances = {} for i, name in enumerate(evaluator.cs.get_hyperparameter_names()): imp = evaluator.quantify_importance((i, )) imp = imp[(i, )]["individual importance"] individual_importances[name] = imp tot_importance = sum(v for v in individual_importances.values()) for name in individual_importances.keys(): individual_importances[name] /= tot_importance param_importances = OrderedDict( reversed( sorted( individual_importances.items(), key=lambda name_and_importance: name_and_importance[1], ))) return param_importances
def evaluate(self, study: Study, params: Optional[List[str]] = None) -> Dict[str, float]: distributions = _get_distributions(study, params) params_data, values_data = _get_study_data(study, distributions) if params_data.size == 0: # `params` were given but as an empty list. return OrderedDict() params_data, cols_to_raw_cols = _encode_categorical( params_data, distributions.values()) forest = self._forest forest.fit(params_data, values_data) feature_importances = forest.feature_importances_ feature_importances_reduced = numpy.zeros(len(distributions)) numpy.add.at(feature_importances_reduced, cols_to_raw_cols, feature_importances) param_importances = OrderedDict() param_names = list(distributions.keys()) for i in feature_importances_reduced.argsort()[::-1]: param_importances[ param_names[i]] = feature_importances_reduced[i].item() return param_importances
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params) if len(distributions) == 0: return OrderedDict() trials = [] for trial in _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target): if any(name not in trial.params for name in distributions.keys()): continue trials.append(trial) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) n_trials = len(trials) self._trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64) self._trans_values = numpy.empty(n_trials, dtype=numpy.float64) for trial_idx, trial in enumerate(trials): self._trans_params[trial_idx] = trans.transform(trial.params) self._trans_values[ trial_idx] = trial.value if target is None else target(trial) encoded_column_to_column = trans.encoded_column_to_column if self._trans_params.size == 0: # `params` were given but as an empty list. return OrderedDict() forest = self._forest forest.fit(self._trans_params, self._trans_values) feature_importances = forest.feature_importances_ feature_importances_reduced = numpy.zeros(len(distributions)) numpy.add.at(feature_importances_reduced, encoded_column_to_column, feature_importances) param_importances = OrderedDict() self._param_names = list(distributions.keys()) for i in feature_importances_reduced.argsort()[::-1]: param_importances[ self._param_names[i]] = feature_importances_reduced[i].item() return param_importances
def evaluate(self, study: Study, params: Optional[List[str]] = None) -> Dict[str, float]: distributions = _get_distributions(study, params) params_data, values_data = _get_study_data(study, distributions) if params_data.size == 0: # `params` were given but as an empty list. return OrderedDict() # Many (deep) copies of the search spaces are required during the tree traversal and using # Optuna distributions will create a bottleneck. # Therefore, search spaces (parameter distributions) are represented by a single # `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical # or not. search_spaces = numpy.empty((len(distributions), 2), dtype=numpy.float64) search_spaces_is_categorical = [] for i, distribution in enumerate(distributions.values()): if isinstance(distribution, CategoricalDistribution): search_spaces[i, 0] = 0 search_spaces[i, 1] = len(distribution.choices) search_spaces_is_categorical.append(True) elif isinstance( distribution, ( DiscreteUniformDistribution, IntLogUniformDistribution, IntUniformDistribution, LogUniformDistribution, UniformDistribution, ), ): search_spaces[i, 0] = distribution.low search_spaces[i, 1] = distribution.high search_spaces_is_categorical.append(False) else: assert False evaluator = self._evaluator evaluator.fit( X=params_data, y=values_data, search_spaces=search_spaces, search_spaces_is_categorical=search_spaces_is_categorical, ) importances = {} for i, name in enumerate(distributions.keys()): importance, _ = evaluator.get_importance((i,)) importances[name] = importance total_importance = sum(importances.values()) for name in importances.keys(): importances[name] /= total_importance sorted_importances = OrderedDict( reversed( sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1]) ) ) return sorted_importances
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params=params) if params is None: params = list(distributions.keys()) assert params is not None if len(params) == 0: return OrderedDict() trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) trans_params: np.ndarray = _get_trans_params(trials, trans) target_values: np.ndarray = _get_target_values(trials, target) forest = self._forest forest.fit(X=trans_params, y=target_values) # Create Tree Explainer object that can calculate shap values. explainer = TreeExplainer(forest) # Generate SHAP values for the parameters during the trials. feature_shap_values: np.ndarray = explainer.shap_values(trans_params) param_shap_values = np.zeros((len(trials), len(params))) np.add.at(param_shap_values.T, trans.encoded_column_to_column, feature_shap_values.T) # Calculate the mean absolute SHAP value for each parameter. # List of tuples ("feature_name": mean_abs_shap_value). mean_abs_shap_values = np.abs(param_shap_values).mean(axis=0) return _sort_dict_by_importance(_param_importances_to_dict(params, mean_abs_shap_values))
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params=params) if params is None: params = list(distributions.keys()) assert params is not None if len(params) == 0: return OrderedDict() trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) trans_params: numpy.ndarray = _get_trans_params(trials, trans) target_values: numpy.ndarray = _get_target_values(trials, target) forest = self._forest forest.fit(X=trans_params, y=target_values) feature_importances = forest.feature_importances_ # Untransform feature importances to param importances # by adding up relevant feature importances. param_importances = numpy.zeros(len(params)) numpy.add.at(param_importances, trans.encoded_column_to_column, feature_importances) return _sort_dict_by_importance(_param_importances_to_dict(params, param_importances))
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`." ) distributions = _get_distributions(study, params) if len(distributions) == 0: return OrderedDict() trials = [] for trial in study.trials: if trial.state != TrialState.COMPLETE: continue if any(name not in trial.params for name in distributions.keys()): continue trials.append(trial) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) n_trials = len(trials) trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64) trans_values = numpy.empty(n_trials, dtype=numpy.float64) for trial_idx, trial in enumerate(trials): trans_params[trial_idx] = trans.transform(trial.params) trans_values[trial_idx] = trial.value if target is None else target(trial) trans_bounds = trans.bounds column_to_encoded_columns = trans.column_to_encoded_columns if trans_params.size == 0: # `params` were given but as an empty list. return OrderedDict() # Many (deep) copies of the search spaces are required during the tree traversal and using # Optuna distributions will create a bottleneck. # Therefore, search spaces (parameter distributions) are represented by a single # `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical # or not. evaluator = self._evaluator evaluator.fit( X=trans_params, y=trans_values, search_spaces=trans_bounds, column_to_encoded_columns=column_to_encoded_columns, ) importances = {} for i, name in enumerate(distributions.keys()): importance, _ = evaluator.get_importance((i,)) importances[name] = importance total_importance = sum(importances.values()) for name in importances.keys(): importances[name] /= total_importance sorted_importances = OrderedDict( reversed( sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1]) ) ) return sorted_importances
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params) if len(distributions) == 0: return OrderedDict() # fANOVA does not support parameter distributions with a single value. # However, there is no reason to calculate parameter importance in such case anyway, # since it will always be 0 as the parameter is constant in the objective function. zero_importances = {name: 0.0 for name, dist in distributions.items() if dist.single()} distributions = {name: dist for name, dist in distributions.items() if not dist.single()} trials = [] for trial in _filter_nonfinite( study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target ): if any(name not in trial.params for name in distributions.keys()): continue trials.append(trial) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) n_trials = len(trials) trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64) trans_values = numpy.empty(n_trials, dtype=numpy.float64) for trial_idx, trial in enumerate(trials): trans_params[trial_idx] = trans.transform(trial.params) trans_values[trial_idx] = trial.value if target is None else target(trial) trans_bounds = trans.bounds column_to_encoded_columns = trans.column_to_encoded_columns if trans_params.size == 0: # `params` were given but as an empty list. return OrderedDict() # Many (deep) copies of the search spaces are required during the tree traversal and using # Optuna distributions will create a bottleneck. # Therefore, search spaces (parameter distributions) are represented by a single # `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical # or not. evaluator = self._evaluator evaluator.fit( X=trans_params, y=trans_values, search_spaces=trans_bounds, column_to_encoded_columns=column_to_encoded_columns, ) importances = {} for i, name in enumerate(distributions.keys()): importance, _ = evaluator.get_importance(i) importances[name] = importance importances = {**importances, **zero_importances} total_importance = sum(importances.values()) for name in importances: importances[name] /= total_importance sorted_importances = OrderedDict( reversed( sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1]) ) ) return sorted_importances
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params=params) if params is None: params = list(distributions.keys()) assert params is not None # fANOVA does not support parameter distributions with a single value. # However, there is no reason to calculate parameter importance in such case anyway, # since it will always be 0 as the parameter is constant in the objective function. non_single_distributions = { name: dist for name, dist in distributions.items() if not dist.single() } single_distributions = { name: dist for name, dist in distributions.items() if dist.single() } if len(non_single_distributions) == 0: return OrderedDict() trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target) trans = _SearchSpaceTransform(non_single_distributions, transform_log=False, transform_step=False) trans_params: numpy.ndarray = _get_trans_params(trials, trans) target_values: numpy.ndarray = _get_target_values(trials, target) evaluator = self._evaluator evaluator.fit( X=trans_params, y=target_values, search_spaces=trans.bounds, column_to_encoded_columns=trans.column_to_encoded_columns, ) param_importances = numpy.array([ evaluator.get_importance(i)[0] for i in range(len(non_single_distributions)) ]) param_importances /= numpy.sum(param_importances) return _sort_dict_by_importance({ **_param_importances_to_dict(non_single_distributions.keys(), param_importances), **_param_importances_to_dict(single_distributions.keys(), 0.0), })