def evaluate(self, study: Study,
params: Optional[List[str]]) -> Dict[str, float]:
distributions = _get_distributions(study, params)
params_data, values_data = _get_study_data(study, distributions)
evaluator = fANOVA(
X=params_data,
Y=values_data,
config_space=_get_configuration_space(distributions),
max_features=max(1, int(params_data.shape[1] * 0.7)),
)
individual_importances = {}
for i, name in enumerate(evaluator.cs.get_hyperparameter_names()):
imp = evaluator.quantify_importance((i, ))
imp = imp[(i, )]["individual importance"]
individual_importances[name] = imp
tot_importance = sum(v for v in individual_importances.values())
for name in individual_importances.keys():
individual_importances[name] /= tot_importance
param_importances = OrderedDict(
reversed(
sorted(
individual_importances.items(),
key=lambda name_and_importance: name_and_importance[1],
)))
return param_importances
def evaluate(self,
study: Study,
params: Optional[List[str]] = None) -> Dict[str, float]:
distributions = _get_distributions(study, params)
params_data, values_data = _get_study_data(study, distributions)
if params_data.size == 0: # `params` were given but as an empty list.
return OrderedDict()
params_data, cols_to_raw_cols = _encode_categorical(
params_data, distributions.values())
forest = self._forest
forest.fit(params_data, values_data)
feature_importances = forest.feature_importances_
feature_importances_reduced = numpy.zeros(len(distributions))
numpy.add.at(feature_importances_reduced, cols_to_raw_cols,
feature_importances)
param_importances = OrderedDict()
param_names = list(distributions.keys())
for i in feature_importances_reduced.argsort()[::-1]:
param_importances[
param_names[i]] = feature_importances_reduced[i].item()
return param_importances
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
trials = []
for trial in _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions,
transform_log=False,
transform_step=False)
n_trials = len(trials)
self._trans_params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
self._trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
self._trans_params[trial_idx] = trans.transform(trial.params)
self._trans_values[
trial_idx] = trial.value if target is None else target(trial)
encoded_column_to_column = trans.encoded_column_to_column
if self._trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
forest = self._forest
forest.fit(self._trans_params, self._trans_values)
feature_importances = forest.feature_importances_
feature_importances_reduced = numpy.zeros(len(distributions))
numpy.add.at(feature_importances_reduced, encoded_column_to_column,
feature_importances)
param_importances = OrderedDict()
self._param_names = list(distributions.keys())
for i in feature_importances_reduced.argsort()[::-1]:
param_importances[
self._param_names[i]] = feature_importances_reduced[i].item()
return param_importances
def evaluate(self, study: Study, params: Optional[List[str]] = None) -> Dict[str, float]:
distributions = _get_distributions(study, params)
params_data, values_data = _get_study_data(study, distributions)
if params_data.size == 0: # `params` were given but as an empty list.
return OrderedDict()
# Many (deep) copies of the search spaces are required during the tree traversal and using
# Optuna distributions will create a bottleneck.
# Therefore, search spaces (parameter distributions) are represented by a single
# `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical
# or not.
search_spaces = numpy.empty((len(distributions), 2), dtype=numpy.float64)
search_spaces_is_categorical = []
for i, distribution in enumerate(distributions.values()):
if isinstance(distribution, CategoricalDistribution):
search_spaces[i, 0] = 0
search_spaces[i, 1] = len(distribution.choices)
search_spaces_is_categorical.append(True)
elif isinstance(
distribution,
(
DiscreteUniformDistribution,
IntLogUniformDistribution,
IntUniformDistribution,
LogUniformDistribution,
UniformDistribution,
),
):
search_spaces[i, 0] = distribution.low
search_spaces[i, 1] = distribution.high
search_spaces_is_categorical.append(False)
else:
assert False
evaluator = self._evaluator
evaluator.fit(
X=params_data,
y=values_data,
search_spaces=search_spaces,
search_spaces_is_categorical=search_spaces_is_categorical,
)
importances = {}
for i, name in enumerate(distributions.keys()):
importance, _ = evaluator.get_importance((i,))
importances[name] = importance
total_importance = sum(importances.values())
for name in importances.keys():
importances[name] /= total_importance
sorted_importances = OrderedDict(
reversed(
sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1])
)
)
return sorted_importances
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params=params)
if params is None:
params = list(distributions.keys())
assert params is not None
if len(params) == 0:
return OrderedDict()
trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
trans_params: np.ndarray = _get_trans_params(trials, trans)
target_values: np.ndarray = _get_target_values(trials, target)
forest = self._forest
forest.fit(X=trans_params, y=target_values)
# Create Tree Explainer object that can calculate shap values.
explainer = TreeExplainer(forest)
# Generate SHAP values for the parameters during the trials.
feature_shap_values: np.ndarray = explainer.shap_values(trans_params)
param_shap_values = np.zeros((len(trials), len(params)))
np.add.at(param_shap_values.T, trans.encoded_column_to_column, feature_shap_values.T)
# Calculate the mean absolute SHAP value for each parameter.
# List of tuples ("feature_name": mean_abs_shap_value).
mean_abs_shap_values = np.abs(param_shap_values).mean(axis=0)
return _sort_dict_by_importance(_param_importances_to_dict(params, mean_abs_shap_values))
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params=params)
if params is None:
params = list(distributions.keys())
assert params is not None
if len(params) == 0:
return OrderedDict()
trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
trans_params: numpy.ndarray = _get_trans_params(trials, trans)
target_values: numpy.ndarray = _get_target_values(trials, target)
forest = self._forest
forest.fit(X=trans_params, y=target_values)
feature_importances = forest.feature_importances_
# Untransform feature importances to param importances
# by adding up relevant feature importances.
param_importances = numpy.zeros(len(params))
numpy.add.at(param_importances, trans.encoded_column_to_column, feature_importances)
return _sort_dict_by_importance(_param_importances_to_dict(params, param_importances))
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
trials = []
for trial in study.trials:
if trial.state != TrialState.COMPLETE:
continue
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
n_trials = len(trials)
trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64)
trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
trans_params[trial_idx] = trans.transform(trial.params)
trans_values[trial_idx] = trial.value if target is None else target(trial)
trans_bounds = trans.bounds
column_to_encoded_columns = trans.column_to_encoded_columns
if trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
# Many (deep) copies of the search spaces are required during the tree traversal and using
# Optuna distributions will create a bottleneck.
# Therefore, search spaces (parameter distributions) are represented by a single
# `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical
# or not.
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=trans_values,
search_spaces=trans_bounds,
column_to_encoded_columns=column_to_encoded_columns,
)
importances = {}
for i, name in enumerate(distributions.keys()):
importance, _ = evaluator.get_importance((i,))
importances[name] = importance
total_importance = sum(importances.values())
for name in importances.keys():
importances[name] /= total_importance
sorted_importances = OrderedDict(
reversed(
sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1])
)
)
return sorted_importances
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
# fANOVA does not support parameter distributions with a single value.
# However, there is no reason to calculate parameter importance in such case anyway,
# since it will always be 0 as the parameter is constant in the objective function.
zero_importances = {name: 0.0 for name, dist in distributions.items() if dist.single()}
distributions = {name: dist for name, dist in distributions.items() if not dist.single()}
trials = []
for trial in _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
n_trials = len(trials)
trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64)
trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
trans_params[trial_idx] = trans.transform(trial.params)
trans_values[trial_idx] = trial.value if target is None else target(trial)
trans_bounds = trans.bounds
column_to_encoded_columns = trans.column_to_encoded_columns
if trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
# Many (deep) copies of the search spaces are required during the tree traversal and using
# Optuna distributions will create a bottleneck.
# Therefore, search spaces (parameter distributions) are represented by a single
# `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical
# or not.
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=trans_values,
search_spaces=trans_bounds,
column_to_encoded_columns=column_to_encoded_columns,
)
importances = {}
for i, name in enumerate(distributions.keys()):
importance, _ = evaluator.get_importance(i)
importances[name] = importance
importances = {**importances, **zero_importances}
total_importance = sum(importances.values())
for name in importances:
importances[name] /= total_importance
sorted_importances = OrderedDict(
reversed(
sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1])
)
)
return sorted_importances
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params=params)
if params is None:
params = list(distributions.keys())
assert params is not None
# fANOVA does not support parameter distributions with a single value.
# However, there is no reason to calculate parameter importance in such case anyway,
# since it will always be 0 as the parameter is constant in the objective function.
non_single_distributions = {
name: dist
for name, dist in distributions.items() if not dist.single()
}
single_distributions = {
name: dist
for name, dist in distributions.items() if dist.single()
}
if len(non_single_distributions) == 0:
return OrderedDict()
trials: List[FrozenTrial] = _get_filtered_trials(study,
params=params,
target=target)
trans = _SearchSpaceTransform(non_single_distributions,
transform_log=False,
transform_step=False)
trans_params: numpy.ndarray = _get_trans_params(trials, trans)
target_values: numpy.ndarray = _get_target_values(trials, target)
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=target_values,
search_spaces=trans.bounds,
column_to_encoded_columns=trans.column_to_encoded_columns,
)
param_importances = numpy.array([
evaluator.get_importance(i)[0]
for i in range(len(non_single_distributions))
])
param_importances /= numpy.sum(param_importances)
return _sort_dict_by_importance({
**_param_importances_to_dict(non_single_distributions.keys(), param_importances),
**_param_importances_to_dict(single_distributions.keys(), 0.0),
})
