def _raise_error_if_multi_objective(self, study: Study) -> None:
if study._is_multi_objective():
raise ValueError(
"If the study is being used for multi-objective optimization, "
f"{self.__class__.__name__} cannot be used."
)
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
trials = []
for trial in _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions,
transform_log=False,
transform_step=False)
n_trials = len(trials)
self._trans_params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
self._trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
self._trans_params[trial_idx] = trans.transform(trial.params)
self._trans_values[
trial_idx] = trial.value if target is None else target(trial)
encoded_column_to_column = trans.encoded_column_to_column
if self._trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
forest = self._forest
forest.fit(self._trans_params, self._trans_values)
feature_importances = forest.feature_importances_
feature_importances_reduced = numpy.zeros(len(distributions))
numpy.add.at(feature_importances_reduced, encoded_column_to_column,
feature_importances)
param_importances = OrderedDict()
self._param_names = list(distributions.keys())
for i in feature_importances_reduced.argsort()[::-1]:
param_importances[
self._param_names[i]] = feature_importances_reduced[i].item()
return param_importances
def plot_optimization_history(
study: Study,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
"""Plot optimization history of all trials in a study with Matplotlib.
.. seealso::
Please refer to :func:`optuna.visualization.plot_optimization_history` for an example.
Example:
The following code snippet shows how to plot optimization history.
.. plot::
import optuna
def objective(trial):
x = trial.suggest_uniform("x", -100, 100)
y = trial.suggest_categorical("y", [-1, 0, 1])
return x ** 2 + y
sampler = optuna.samplers.TPESampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=10)
optuna.visualization.matplotlib.plot_optimization_history(study)
Args:
study:
A :class:`~optuna.study.Study` object whose trials are plotted for their target values.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective optimization.
target_name:
Target's name to display on the axis label and the legend.
Returns:
A :class:`matplotlib.axes.Axes` object.
Raises:
:exc:`ValueError`:
If ``target`` is :obj:`None` and ``study`` is being used for multi-objective
optimization.
"""
_imports.check()
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`.")
return _get_optimization_history_plot(study, target, target_name)
def sample_independent(
self,
study: Study,
trial: FrozenTrial,
param_name: str,
param_distribution: BaseDistribution,
) -> Any:
values, scores, violations = _get_observation_pairs(
study,
[param_name],
self._multivariate,
self._constant_liar,
self._constraints_func is not None,
)
n = len(scores)
self._log_independent_sampling(n, trial, param_name)
if n < self._n_startup_trials:
return self._random_sampler.sample_independent(
study, trial, param_name, param_distribution)
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n), violations)
# `None` items are intentionally converted to `nan` and then filtered out.
# For `nan` conversion, the dtype must be float.
config_values = {
k: np.asarray(v, dtype=float)
for k, v in values.items()
}
below = _build_observation_dict(config_values, indices_below)
above = _build_observation_dict(config_values, indices_above)
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
config_values, scores, indices_below, violations)
mpe_below = _ParzenEstimator(
below,
{param_name: param_distribution},
self._parzen_estimator_parameters,
weights_below,
)
else:
mpe_below = _ParzenEstimator(below,
{param_name: param_distribution},
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, {param_name: param_distribution},
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
return param_distribution.to_external_repr(ret[param_name])
def _sample_relative(
self, study: Study, trial: FrozenTrial,
search_space: Dict[str, BaseDistribution]) -> Dict[str, Any]:
if search_space == {}:
return {}
param_names = list(search_space.keys())
values, scores, violations = _get_observation_pairs(
study,
param_names,
self._multivariate,
self._constant_liar,
self._constraints_func is not None,
)
# If the number of samples is insufficient, we run random trial.
n = len(scores)
if n < self._n_startup_trials:
return {}
# We divide data into below and above.
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n), violations)
# `None` items are intentionally converted to `nan` and then filtered out.
# For `nan` conversion, the dtype must be float.
config_values = {
k: np.asarray(v, dtype=float)
for k, v in values.items()
}
below = _build_observation_dict(config_values, indices_below)
above = _build_observation_dict(config_values, indices_above)
# We then sample by maximizing log likelihood ratio.
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
config_values, scores, indices_below, violations)
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters,
weights_below)
else:
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, search_space,
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
for param_name, dist in search_space.items():
ret[param_name] = dist.to_external_repr(ret[param_name])
return ret
def crossover(
self,
parents_params: np.ndarray,
rng: np.random.RandomState,
study: Study,
search_space_bounds: np.ndarray,
) -> np.ndarray:
# https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.422.952&rep=rep1&type=pdf
# Section 3.2 Crossover Schemes (vSBX)
if self._eta is None:
eta = 20.0 if study._is_multi_objective() else 2.0
else:
eta = self._eta
us = rng.rand(len(search_space_bounds))
beta_1 = np.power(1 / 2 * us, 1 / (eta + 1))
beta_2 = np.power(1 / 2 * (1 - us), 1 / (eta + 1))
mask = us > 0.5
c1 = 0.5 * ((1 + beta_1) * parents_params[0] +
(1 - beta_1) * parents_params[1])
c1[mask] = (0.5 * ((1 - beta_1) * parents_params[0] +
(1 + beta_1) * parents_params[1])[mask])
c2 = 0.5 * ((3 - beta_2) * parents_params[0] -
(1 - beta_2) * parents_params[1])
c2[mask] = (0.5 * (-(1 - beta_2) * parents_params[0] +
(3 - beta_2) * parents_params[1])[mask])
# vSBX applies crossover with establishment 0.5, and with probability 0.5,
# the gene of the parent individual is the gene of the child individual.
# The original SBX creates two child individuals,
# but optuna's implementation creates only one child individual.
# Therefore, when there is no crossover,
# the gene is selected with equal probability from the parent individuals x1 and x2.
child_params_list = []
for c1_i, c2_i, x1_i, x2_i in zip(c1, c2, parents_params[0],
parents_params[1]):
if rng.rand() < 0.5:
if rng.rand() < 0.5:
child_params_list.append(c1_i)
else:
child_params_list.append(c2_i)
else:
if rng.rand() < 0.5:
child_params_list.append(x1_i)
else:
child_params_list.append(x2_i)
child_params = np.array(child_params_list)
return child_params
def _sample_relative(
self, study: Study, trial: FrozenTrial,
search_space: Dict[str, BaseDistribution]) -> Dict[str, Any]:
if search_space == {}:
return {}
param_names = list(search_space.keys())
values, scores = _get_observation_pairs(study, param_names,
self._multivariate,
self._constant_liar)
# If the number of samples is insufficient, we run random trial.
n = len(scores)
if n < self._n_startup_trials:
return {}
# We divide data into below and above.
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n))
below = _build_observation_dict(values, indices_below)
above = _build_observation_dict(values, indices_above)
# We then sample by maximizing log likelihood ratio.
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
values, scores, indices_below)
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters,
weights_below)
else:
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, search_space,
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
for param_name, dist in search_space.items():
ret[param_name] = dist.to_external_repr(ret[param_name])
return ret
def sample_independent(
self,
study: Study,
trial: FrozenTrial,
param_name: str,
param_distribution: BaseDistribution,
) -> Any:
values, scores = _get_observation_pairs(study, [param_name],
self._multivariate,
self._constant_liar)
n = len(scores)
if n < self._n_startup_trials:
return self._random_sampler.sample_independent(
study, trial, param_name, param_distribution)
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n))
below = _build_observation_dict(values, indices_below)
above = _build_observation_dict(values, indices_above)
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
values, scores, indices_below)
mpe_below = _ParzenEstimator(
below,
{param_name: param_distribution},
self._parzen_estimator_parameters,
weights_below,
)
else:
mpe_below = _ParzenEstimator(below,
{param_name: param_distribution},
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, {param_name: param_distribution},
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
return param_distribution.to_external_repr(ret[param_name])
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params=params)
if params is None:
params = list(distributions.keys())
assert params is not None
if len(params) == 0:
return OrderedDict()
trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
trans_params: np.ndarray = _get_trans_params(trials, trans)
target_values: np.ndarray = _get_target_values(trials, target)
forest = self._forest
forest.fit(X=trans_params, y=target_values)
# Create Tree Explainer object that can calculate shap values.
explainer = TreeExplainer(forest)
# Generate SHAP values for the parameters during the trials.
feature_shap_values: np.ndarray = explainer.shap_values(trans_params)
param_shap_values = np.zeros((len(trials), len(params)))
np.add.at(param_shap_values.T, trans.encoded_column_to_column, feature_shap_values.T)
# Calculate the mean absolute SHAP value for each parameter.
# List of tuples ("feature_name": mean_abs_shap_value).
mean_abs_shap_values = np.abs(param_shap_values).mean(axis=0)
return _sort_dict_by_importance(_param_importances_to_dict(params, mean_abs_shap_values))
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params=params)
if params is None:
params = list(distributions.keys())
assert params is not None
if len(params) == 0:
return OrderedDict()
trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
trans_params: numpy.ndarray = _get_trans_params(trials, trans)
target_values: numpy.ndarray = _get_target_values(trials, target)
forest = self._forest
forest.fit(X=trans_params, y=target_values)
feature_importances = forest.feature_importances_
# Untransform feature importances to param importances
# by adding up relevant feature importances.
param_importances = numpy.zeros(len(params))
numpy.add.at(param_importances, trans.encoded_column_to_column, feature_importances)
return _sort_dict_by_importance(_param_importances_to_dict(params, param_importances))
def _get_observation_pairs(
study: Study,
param_names: List[str],
multivariate: bool,
constant_liar:
bool = False, # TODO(hvy): Remove default value and fix unit tests.
constraints_enabled: bool = False,
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, List[float]]],
Optional[List[float]], ]:
"""Get observation pairs from the study.
This function collects observation pairs from the complete or pruned trials of the study.
In addition, if ``constant_liar`` is :obj:`True`, the running trials are considered.
The values for trials that don't contain the parameter in the ``param_names`` are skipped.
An observation pair fundamentally consists of a parameter value and an objective value.
However, due to the pruning mechanism of Optuna, final objective values are not always
available. Therefore, this function uses intermediate values in addition to the final
ones, and reports the value with its step count as ``(-step, value)``.
Consequently, the structure of the observation pair is as follows:
``(param_value, (-step, value))``.
The second element of an observation pair is used to rank observations in
``_split_observation_pairs`` method (i.e., observations are sorted lexicographically by
``(-step, value)``).
When ``constraints_enabled`` is :obj:`True`, 1-dimensional violation values are returned
as the third element (:obj:`None` otherwise). Each value is a float of 0 or greater and a
trial is feasible if and only if its violation score is 0.
"""
if len(param_names) > 1:
assert multivariate
signs = []
for d in study.directions:
if d == StudyDirection.MINIMIZE:
signs.append(1)
else:
signs.append(-1)
states: Container[TrialState]
if constant_liar:
states = (TrialState.COMPLETE, TrialState.PRUNED, TrialState.RUNNING)
else:
states = (TrialState.COMPLETE, TrialState.PRUNED)
scores = []
values: Dict[str, List[Optional[float]]] = {
param_name: []
for param_name in param_names
}
violations: Optional[List[float]] = [] if constraints_enabled else None
for trial in study.get_trials(deepcopy=False, states=states):
# If ``multivariate`` = True and ``group`` = True, we ignore the trials that are not
# included in each subspace.
# If ``multivariate`` = False, we skip the check.
if multivariate and any(
[param_name not in trial.params for param_name in param_names]):
continue
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.values is None:
continue
score = (-float("inf"),
[sign * v for sign, v in zip(signs, trial.values)])
elif trial.state is TrialState.PRUNED:
if study._is_multi_objective():
continue
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(
trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, [float("inf")])
else:
score = (-step, [signs[0] * intermediate_value])
else:
score = (float("inf"), [0.0])
elif trial.state is TrialState.RUNNING:
if study._is_multi_objective():
continue
assert constant_liar
score = (-float("inf"), [signs[0] * float("inf")])
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
param_value: Optional[float]
if param_name in trial.params:
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(
trial.params[param_name])
else:
param_value = None
values[param_name].append(param_value)
if constraints_enabled:
assert violations is not None
constraint = trial.system_attrs.get(_CONSTRAINTS_KEY)
if constraint is None:
warnings.warn(
f"Trial {trial.number} does not have constraint values."
" It will be treated as a lower priority than other trials."
)
violation = float("inf")
else:
# Violation values of infeasible dimensions are summed up.
violation = sum(v for v in constraint if v > 0)
violations.append(violation)
return values, scores, violations
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params=params)
if params is None:
params = list(distributions.keys())
assert params is not None
# fANOVA does not support parameter distributions with a single value.
# However, there is no reason to calculate parameter importance in such case anyway,
# since it will always be 0 as the parameter is constant in the objective function.
non_single_distributions = {
name: dist
for name, dist in distributions.items() if not dist.single()
}
single_distributions = {
name: dist
for name, dist in distributions.items() if dist.single()
}
if len(non_single_distributions) == 0:
return OrderedDict()
trials: List[FrozenTrial] = _get_filtered_trials(study,
params=params,
target=target)
trans = _SearchSpaceTransform(non_single_distributions,
transform_log=False,
transform_step=False)
trans_params: numpy.ndarray = _get_trans_params(trials, trans)
target_values: numpy.ndarray = _get_target_values(trials, target)
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=target_values,
search_spaces=trans.bounds,
column_to_encoded_columns=trans.column_to_encoded_columns,
)
param_importances = numpy.array([
evaluator.get_importance(i)[0]
for i in range(len(non_single_distributions))
])
param_importances /= numpy.sum(param_importances)
return _sort_dict_by_importance({
**_param_importances_to_dict(non_single_distributions.keys(), param_importances),
**_param_importances_to_dict(single_distributions.keys(), 0.0),
})
def plot_contour(
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
"""Plot the parameter relationship as contour plot in a study with Matplotlib.
Note that, if a parameter contains missing values, a trial with missing values is not plotted.
.. seealso::
Please refer to :func:`optuna.visualization.plot_contour` for an example.
Warnings:
Output figures of this Matplotlib-based
:func:`~optuna.visualization.matplotlib.plot_contour` function would be different from
those of the Plotly-based :func:`~optuna.visualization.plot_contour`.
Example:
The following code snippet shows how to plot the parameter relationship as contour plot.
.. plot::
import optuna
def objective(trial):
x = trial.suggest_uniform("x", -100, 100)
y = trial.suggest_categorical("y", [-1, 0, 1])
return x ** 2 + y
sampler = optuna.samplers.TPESampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=30)
optuna.visualization.matplotlib.plot_contour(study, params=["x", "y"])
Args:
study:
A :class:`~optuna.study.Study` object whose trials are plotted for their target values.
params:
Parameter list to visualize. The default is all parameters.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective optimization.
target_name:
Target's name to display on the color bar.
Returns:
A :class:`matplotlib.axes.Axes` object.
Raises:
:exc:`ValueError`:
If ``target`` is :obj:`None` and ``study`` is being used for multi-objective
optimization.
"""
_imports.check()
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`.")
_logger.warning(
"Output figures of this Matplotlib-based `plot_contour` function would be different from "
"those of the Plotly-based `plot_contour`.")
return _get_contour_plot(study, params, target, target_name)
def plot_parallel_coordinate(
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
"""Plot the high-dimentional parameter relationships in a study.
Note that, If a parameter contains missing values, a trial with missing values is not plotted.
Example:
The following code snippet shows how to plot the high-dimentional parameter relationships.
.. plotly::
import optuna
def objective(trial):
x = trial.suggest_uniform("x", -100, 100)
y = trial.suggest_categorical("y", [-1, 0, 1])
return x ** 2 + y
sampler = optuna.samplers.TPESampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=10)
optuna.visualization.plot_parallel_coordinate(study, params=["x", "y"])
Args:
study:
A :class:`~optuna.study.Study` object whose trials are plotted for their target values.
params:
Parameter list to visualize. The default is all parameters.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective optimization.
target_name:
Target's name to display on the axis label and the legend.
Returns:
A :class:`plotly.graph_objs.Figure` object.
Raises:
:exc:`ValueError`:
If ``target`` is :obj:`None` and ``study`` is being used for multi-objective
optimization.
"""
_imports.check()
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`.")
return _get_parallel_coordinate_plot(study, params, target, target_name)
def crossover(
self,
parents_params: np.ndarray,
rng: np.random.RandomState,
study: Study,
search_space_bounds: np.ndarray,
) -> np.ndarray:
# https://www.researchgate.net/profile/M-M-Raghuwanshi/publication/267198495_Simulated_Binary_Crossover_with_Lognormal_Distribution/links/5576c78408ae7536375205d7/Simulated-Binary-Crossover-with-Lognormal-Distribution.pdf
# Section 2 Simulated Binary Crossover (SBX)
# To avoid generating solutions that violate the box constraints,
# alpha1, alpha2, xls and xus are introduced, unlike the reference.
xls = search_space_bounds[..., 0]
xus = search_space_bounds[..., 1]
xs_min = np.min(parents_params, axis=0)
xs_max = np.max(parents_params, axis=0)
if self._eta is None:
eta = 20.0 if study._is_multi_objective() else 2.0
else:
eta = self._eta
xs_diff = np.clip(xs_max - xs_min, 1e-10, None)
beta1 = 1 + 2 * (xs_min - xls) / xs_diff
beta2 = 1 + 2 * (xus - xs_max) / xs_diff
alpha1 = 2 - np.power(beta1, -(eta + 1))
alpha2 = 2 - np.power(beta2, -(eta + 1))
us = rng.rand(len(search_space_bounds))
mask1 = us > 1 / alpha1 # Equation (3).
betaq1 = np.power(us * alpha1, 1 / (eta + 1)) # Equation (3).
betaq1[mask1] = np.power((1 / (2 - us * alpha1)), 1 / (eta + 1))[mask1] # Equation (3).
mask2 = us > 1 / alpha2 # Equation (3).
betaq2 = np.power(us * alpha2, 1 / (eta + 1)) # Equation (3)
betaq2[mask2] = np.power((1 / (2 - us * alpha2)), 1 / (eta + 1))[mask2] # Equation (3).
c1 = 0.5 * ((xs_min + xs_max) - betaq1 * xs_diff) # Equation (4).
c2 = 0.5 * ((xs_min + xs_max) + betaq2 * xs_diff) # Equation (5).
# SBX applies crossover with establishment 0.5, and with probability 0.5,
# the gene of the parent individual is the gene of the child individual.
# The original SBX creates two child individuals,
# but optuna's implementation creates only one child individual.
# Therefore, when there is no crossover,
# the gene is selected with equal probability from the parent individuals x1 and x2.
child_params_list = []
for c1_i, c2_i, x1_i, x2_i in zip(c1, c2, parents_params[0], parents_params[1]):
if rng.rand() < 0.5:
if rng.rand() < 0.5:
child_params_list.append(c1_i)
else:
child_params_list.append(c2_i)
else:
if rng.rand() < 0.5:
child_params_list.append(x1_i)
else:
child_params_list.append(x2_i)
child_params = np.array(child_params_list)
return child_params
def plot_param_importances(
study: Study,
evaluator: Optional[BaseImportanceEvaluator] = None,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
"""Plot hyperparameter importances.
Example:
The following code snippet shows how to plot hyperparameter importances.
.. plotly::
import optuna
def objective(trial):
x = trial.suggest_int("x", 0, 2)
y = trial.suggest_float("y", -1.0, 1.0)
z = trial.suggest_float("z", 0.0, 1.5)
return x ** 2 + y ** 3 - z ** 4
sampler = optuna.samplers.RandomSampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=100)
optuna.visualization.plot_param_importances(study)
.. seealso::
This function visualizes the results of :func:`optuna.importance.get_param_importances`.
Args:
study:
An optimized study.
evaluator:
An importance evaluator object that specifies which algorithm to base the importance
assessment on.
Defaults to
:class:`~optuna.importance.FanovaImportanceEvaluator`.
params:
A list of names of parameters to assess.
If :obj:`None`, all parameters that are present in all of the completed trials are
assessed.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective optimization.
target_name:
Target's name to display on the axis label.
Returns:
A :class:`plotly.graph_objs.Figure` object.
Raises:
:exc:`ValueError`:
If ``target`` is :obj:`None` and ``study`` is being used for multi-objective
optimization.
"""
_imports.check()
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`."
)
layout = go.Layout(
title="Hyperparameter Importances",
xaxis={"title": f"Importance for {target_name}"},
yaxis={"title": "Hyperparameter"},
showlegend=False,
)
# Importances cannot be evaluated without completed trials.
# Return an empty figure for consistency with other visualization functions.
trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE]
if len(trials) == 0:
logger.warning("Study instance does not contain completed trials.")
return go.Figure(data=[], layout=layout)
importances = optuna.importance.get_param_importances(
study, evaluator=evaluator, params=params, target=target
)
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
fig = go.Figure(
data=[
go.Bar(
x=importance_values,
y=param_names,
text=importance_values,
texttemplate="%{text:.2f}",
textposition="outside",
cliponaxis=False, # Ensure text is not clipped.
hovertemplate=[
_make_hovertext(param_name, importance, study)
for param_name, importance in importances.items()
],
marker_color=[_get_color(param_name, study) for param_name in param_names],
orientation="h",
)
],
layout=layout,
)
return fig
def _get_observation_pairs(
study: Study,
param_names: List[str],
multivariate: bool,
constant_liar: bool = False, # TODO(hvy): Remove default value and fix unit tests.
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, List[float]]]]:
"""Get observation pairs from the study.
This function collects observation pairs from the complete or pruned trials of the study.
In addition, if ``constant_liar`` is :obj:`True`, the running trials are considered.
The values for trials that don't contain the parameter in the ``param_names`` are skipped.
An observation pair fundamentally consists of a parameter value and an objective value.
However, due to the pruning mechanism of Optuna, final objective values are not always
available. Therefore, this function uses intermediate values in addition to the final
ones, and reports the value with its step count as ``(-step, value)``.
Consequently, the structure of the observation pair is as follows:
``(param_value, (-step, value))``.
The second element of an observation pair is used to rank observations in
``_split_observation_pairs`` method (i.e., observations are sorted lexicographically by
``(-step, value)``).
"""
if len(param_names) > 1:
assert multivariate
signs = []
for d in study.directions:
if d == StudyDirection.MINIMIZE:
signs.append(1)
else:
signs.append(-1)
states: Tuple[TrialState, ...]
if constant_liar:
states = (TrialState.COMPLETE, TrialState.PRUNED, TrialState.RUNNING)
else:
states = (TrialState.COMPLETE, TrialState.PRUNED)
scores = []
values: Dict[str, List[Optional[float]]] = {param_name: [] for param_name in param_names}
for trial in study.get_trials(deepcopy=False, states=states):
# If ``multivariate`` = True and ``group`` = True, we ignore the trials that are not
# included in each subspace.
# If ``multivariate`` = False, we skip the check.
if multivariate and any([param_name not in trial.params for param_name in param_names]):
continue
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.values is None:
continue
score = (-float("inf"), [sign * v for sign, v in zip(signs, trial.values)])
elif trial.state is TrialState.PRUNED:
if study._is_multi_objective():
continue
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, [float("inf")])
else:
score = (-step, [signs[0] * intermediate_value])
else:
score = (float("inf"), [0.0])
elif trial.state is TrialState.RUNNING:
if study._is_multi_objective():
continue
assert constant_liar
score = (-float("inf"), [signs[0] * float("inf")])
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
raw_param_value = trial.params.get(param_name, None)
param_value: Optional[float]
if raw_param_value is not None:
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(trial.params[param_name])
else:
param_value = None
values[param_name].append(param_value)
return values, scores
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
# fANOVA does not support parameter distributions with a single value.
# However, there is no reason to calculate parameter importance in such case anyway,
# since it will always be 0 as the parameter is constant in the objective function.
zero_importances = {name: 0.0 for name, dist in distributions.items() if dist.single()}
distributions = {name: dist for name, dist in distributions.items() if not dist.single()}
trials = []
for trial in _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
n_trials = len(trials)
trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64)
trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
trans_params[trial_idx] = trans.transform(trial.params)
trans_values[trial_idx] = trial.value if target is None else target(trial)
trans_bounds = trans.bounds
column_to_encoded_columns = trans.column_to_encoded_columns
if trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
# Many (deep) copies of the search spaces are required during the tree traversal and using
# Optuna distributions will create a bottleneck.
# Therefore, search spaces (parameter distributions) are represented by a single
# `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical
# or not.
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=trans_values,
search_spaces=trans_bounds,
column_to_encoded_columns=column_to_encoded_columns,
)
importances = {}
for i, name in enumerate(distributions.keys()):
importance, _ = evaluator.get_importance(i)
importances[name] = importance
importances = {**importances, **zero_importances}
total_importance = sum(importances.values())
for name in importances:
importances[name] /= total_importance
sorted_importances = OrderedDict(
reversed(
sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1])
)
)
return sorted_importances
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
trials = []
for trial in study.trials:
if trial.state != TrialState.COMPLETE:
continue
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
n_trials = len(trials)
trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64)
trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
trans_params[trial_idx] = trans.transform(trial.params)
trans_values[trial_idx] = trial.value if target is None else target(trial)
trans_bounds = trans.bounds
column_to_encoded_columns = trans.column_to_encoded_columns
if trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
# Many (deep) copies of the search spaces are required during the tree traversal and using
# Optuna distributions will create a bottleneck.
# Therefore, search spaces (parameter distributions) are represented by a single
# `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical
# or not.
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=trans_values,
search_spaces=trans_bounds,
column_to_encoded_columns=column_to_encoded_columns,
)
importances = {}
for i, name in enumerate(distributions.keys()):
importance, _ = evaluator.get_importance((i,))
importances[name] = importance
total_importance = sum(importances.values())
for name in importances.keys():
importances[name] /= total_importance
sorted_importances = OrderedDict(
reversed(
sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1])
)
)
return sorted_importances
def plot_param_importances(
study: Study,
evaluator: Optional[BaseImportanceEvaluator] = None,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
"""Plot hyperparameter importances with Matplotlib.
.. seealso::
Please refer to :func:`optuna.visualization.plot_param_importances` for an example.
Example:
The following code snippet shows how to plot hyperparameter importances.
.. plot::
import optuna
def objective(trial):
x = trial.suggest_int("x", 0, 2)
y = trial.suggest_float("y", -1.0, 1.0)
z = trial.suggest_float("z", 0.0, 1.5)
return x ** 2 + y ** 3 - z ** 4
sampler = optuna.samplers.RandomSampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=100)
optuna.visualization.matplotlib.plot_param_importances(study)
Args:
study:
An optimized study.
evaluator:
An importance evaluator object that specifies which algorithm to base the importance
assessment on.
Defaults to
:class:`~optuna.importance.FanovaImportanceEvaluator`.
params:
A list of names of parameters to assess.
If :obj:`None`, all parameters that are present in all of the completed trials are
assessed.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective optimization.
target_name:
Target's name to display on the axis label.
Returns:
A :class:`matplotlib.axes.Axes` object.
Raises:
:exc:`ValueError`:
If ``target`` is :obj:`None` and ``study`` is being used for multi-objective
optimization.
"""
_imports.check()
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`.")
return _get_param_importance_plot(study, evaluator, params, target,
target_name)
