def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
trials = []
for trial in _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions,
transform_log=False,
transform_step=False)
n_trials = len(trials)
self._trans_params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
self._trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
self._trans_params[trial_idx] = trans.transform(trial.params)
self._trans_values[
trial_idx] = trial.value if target is None else target(trial)
encoded_column_to_column = trans.encoded_column_to_column
if self._trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
forest = self._forest
forest.fit(self._trans_params, self._trans_values)
feature_importances = forest.feature_importances_
feature_importances_reduced = numpy.zeros(len(distributions))
numpy.add.at(feature_importances_reduced, encoded_column_to_column,
feature_importances)
param_importances = OrderedDict()
self._param_names = list(distributions.keys())
for i in feature_importances_reduced.argsort()[::-1]:
param_importances[
self._param_names[i]] = feature_importances_reduced[i].item()
return param_importances
def after_trial(
self,
study: Study,
trial: FrozenTrial,
state: TrialState,
values: Optional[Sequence[float]],
) -> None:
target_grids = self._get_unvisited_grid_ids(study)
if len(target_grids) == 0:
study.stop()
elif len(target_grids) == 1:
grid_id = study._storage.get_trial_system_attrs(trial._trial_id)["grid_id"]
if grid_id == target_grids[0]:
study.stop()
def _raise_error_if_multi_objective(self, study: Study) -> None:
if study._is_multi_objective():
raise ValueError(
"If the study is being used for multi-objective optimization, "
f"{self.__class__.__name__} cannot be used."
)
def get_param_importances(study_id: int) -> BottleViewReturn:
# TODO(chenghuzi): add support for selecting params and targets via query parameters.
response.content_type = "application/json"
study_name = storage.get_study_name_from_id(study_id)
study = Study(study_name=study_name, storage=storage)
trials = [
trial for trial in study.trials
if trial.state == TrialState.COMPLETE
]
evaluator = None
params = None
target = None
if len(trials) > 0:
importances = optuna.importance.get_param_importances(
study, evaluator=evaluator, params=params, target=target)
else:
importances = {}
target_name = "Objective Value"
return {
"target_name":
target_name,
"param_importances": [{
"name":
name,
"importance":
importance,
"distribution":
get_distribution_name(name, study),
} for name, importance in importances.items()],
}
def infer_relative_search_space(
self, study: Study, trial: FrozenTrial
) -> Dict[str, BaseDistribution]:
if not self._multivariate:
return {}
n_complete_trials = len(study.get_trials(deepcopy=False))
search_space: Dict[str, BaseDistribution] = {}
if self._group:
assert self._group_decomposed_search_space is not None
self._search_space_group = self._group_decomposed_search_space.calculate(study)
for sub_space in self._search_space_group.search_spaces:
for name, distribution in sub_space.items():
if not isinstance(distribution, _DISTRIBUTION_CLASSES):
self._log_independent_sampling(n_complete_trials, trial, name)
continue
search_space[name] = distribution
return search_space
for name, distribution in self._search_space.calculate(study).items():
if not isinstance(distribution, _DISTRIBUTION_CLASSES):
self._log_independent_sampling(n_complete_trials, trial, name)
continue
search_space[name] = distribution
return search_space
def _get_non_pareto_front_trials(
study: Study, pareto_trials: List[FrozenTrial]) -> List[FrozenTrial]:
non_pareto_trials = []
for trial in study.get_trials():
if trial.state == TrialState.COMPLETE and trial not in pareto_trials:
non_pareto_trials.append(trial)
return non_pareto_trials
def plot_optimization_history(
study: Study,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
"""Plot optimization history of all trials in a study with Matplotlib.
.. seealso::
Please refer to :func:`optuna.visualization.plot_optimization_history` for an example.
Example:
The following code snippet shows how to plot optimization history.
.. plot::
import optuna
def objective(trial):
x = trial.suggest_uniform("x", -100, 100)
y = trial.suggest_categorical("y", [-1, 0, 1])
return x ** 2 + y
sampler = optuna.samplers.TPESampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=10)
optuna.visualization.matplotlib.plot_optimization_history(study)
Args:
study:
A :class:`~optuna.study.Study` object whose trials are plotted for their target values.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective optimization.
target_name:
Target's name to display on the axis label and the legend.
Returns:
A :class:`matplotlib.axes.Axes` object.
Raises:
:exc:`ValueError`:
If ``target`` is :obj:`None` and ``study`` is being used for multi-objective
optimization.
"""
_imports.check()
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`.")
return _get_optimization_history_plot(study, target, target_name)
def sample_independent(
self,
study: Study,
trial: FrozenTrial,
param_name: str,
param_distribution: BaseDistribution,
) -> Any:
values, scores, violations = _get_observation_pairs(
study,
[param_name],
self._multivariate,
self._constant_liar,
self._constraints_func is not None,
)
n = len(scores)
self._log_independent_sampling(n, trial, param_name)
if n < self._n_startup_trials:
return self._random_sampler.sample_independent(
study, trial, param_name, param_distribution)
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n), violations)
# `None` items are intentionally converted to `nan` and then filtered out.
# For `nan` conversion, the dtype must be float.
config_values = {
k: np.asarray(v, dtype=float)
for k, v in values.items()
}
below = _build_observation_dict(config_values, indices_below)
above = _build_observation_dict(config_values, indices_above)
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
config_values, scores, indices_below, violations)
mpe_below = _ParzenEstimator(
below,
{param_name: param_distribution},
self._parzen_estimator_parameters,
weights_below,
)
else:
mpe_below = _ParzenEstimator(below,
{param_name: param_distribution},
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, {param_name: param_distribution},
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
return param_distribution.to_external_repr(ret[param_name])
def _get_observation_pairs(
study: Study, param_name: str
) -> Tuple[List[Optional[float]], List[Tuple[float, float]]]:
"""Get observation pairs from the study.
This function collects observation pairs from the complete or pruned trials of the study.
The values for trials that don't contain the parameter named ``param_name`` are set to None.
An observation pair fundamentally consists of a parameter value and an objective value.
However, due to the pruning mechanism of Optuna, final objective values are not always
available. Therefore, this function uses intermediate values in addition to the final
ones, and reports the value with its step count as ``(-step, value)``.
Consequently, the structure of the observation pair is as follows:
``(param_value, (-step, value))``.
The second element of an observation pair is used to rank observations in
``_split_observation_pairs`` method (i.e., observations are sorted lexicographically by
``(-step, value)``).
"""
sign = 1
if study.direction == StudyDirection.MAXIMIZE:
sign = -1
values = []
scores = []
for trial in study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE,
TrialState.PRUNED)):
if trial.state is TrialState.COMPLETE:
if trial.value is None:
continue
score = (-float("inf"), sign * trial.value)
elif trial.state is TrialState.PRUNED:
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(
trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, float("inf"))
else:
score = (-step, sign * intermediate_value)
else:
score = (float("inf"), 0.0)
else:
assert False
param_value: Optional[float] = None
if param_name in trial.params:
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(
trial.params[param_name])
values.append(param_value)
scores.append(score)
return values, scores
def _get_filtered_trials(
study: Study, params: Collection[str], target: Optional[Callable[[FrozenTrial], float]]
) -> List[FrozenTrial]:
trials = study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,))
return [
trial
for trial in trials
if set(params) <= set(trial.params)
and numpy.isfinite(target(trial) if target is not None else cast(float, trial.value))
]
def _sample_relative(
self, study: Study, trial: FrozenTrial,
search_space: Dict[str, BaseDistribution]) -> Dict[str, Any]:
if search_space == {}:
return {}
param_names = list(search_space.keys())
values, scores, violations = _get_observation_pairs(
study,
param_names,
self._multivariate,
self._constant_liar,
self._constraints_func is not None,
)
# If the number of samples is insufficient, we run random trial.
n = len(scores)
if n < self._n_startup_trials:
return {}
# We divide data into below and above.
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n), violations)
# `None` items are intentionally converted to `nan` and then filtered out.
# For `nan` conversion, the dtype must be float.
config_values = {
k: np.asarray(v, dtype=float)
for k, v in values.items()
}
below = _build_observation_dict(config_values, indices_below)
above = _build_observation_dict(config_values, indices_above)
# We then sample by maximizing log likelihood ratio.
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
config_values, scores, indices_below, violations)
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters,
weights_below)
else:
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, search_space,
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
for param_name, dist in search_space.items():
ret[param_name] = dist.to_external_repr(ret[param_name])
return ret
def sample_relative(
self, study: Study, trial: FrozenTrial,
search_space: Dict[str, BaseDistribution]) -> Dict[str, Any]:
# Instead of returning param values, GridSampler puts the target grid id as a system attr,
# and the values are returned from `sample_independent`. This is because the distribution
# object is hard to get at the beginning of trial, while we need the access to the object
# to validate the sampled value.
target_grids = self._get_unvisited_grid_ids(study)
if len(target_grids) == 0:
# This case may occur with distributed optimization or trial queue. If there is no
# target grid, `GridSampler` evaluates a visited, duplicated point with the current
# trial. After that, the optimization stops.
_logger.warning(
"`GridSampler` is re-evaluating a configuration because the grid has been "
"exhausted. This may happen due to a timing issue during distributed optimization "
"or when re-running optimizations on already finished studies."
)
# One of all grids is randomly picked up in this case.
target_grids = list(range(len(self._all_grids)))
study.stop()
elif len(target_grids) == 1:
# When there is only one target grid, optimization stops after the current trial
# finishes.
study.stop()
# In distributed optimization, multiple workers may simultaneously pick up the same grid.
# To make the conflict less frequent, the grid is chosen randomly.
grid_id = random.choice(target_grids)
study._storage.set_trial_system_attr(trial._trial_id, "search_space",
self._search_space)
study._storage.set_trial_system_attr(trial._trial_id, "grid_id",
grid_id)
return {}
def _get_contour_info(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> _ContourInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
sorted_params = []
elif params is None:
sorted_params = sorted(all_params)
else:
if len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
"Parameter {} does not exist in your study.".format(
input_p_name))
sorted_params = sorted(set(params))
sub_plot_infos: List[List[_SubContourInfo]]
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plot_info = _get_contour_subplot_info(trials, x_param, y_param,
target)
sub_plot_infos = [[sub_plot_info]]
else:
sub_plot_infos = []
for i, y_param in enumerate(sorted_params):
sub_plot_infos.append([])
for x_param in sorted_params:
sub_plot_info = _get_contour_subplot_info(
trials, x_param, y_param, target)
sub_plot_infos[i].append(sub_plot_info)
reverse_scale = _is_reverse_scale(study, target)
return _ContourInfo(
sorted_params=sorted_params,
sub_plot_infos=sub_plot_infos,
reverse_scale=reverse_scale,
target_name=target_name,
)
def _get_multivariate_observation_pairs(
study: Study,
param_names: List[str],
constant_liar: bool = False, # TODO(hvy): Remove default value and fix unit tests.
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, float]]]:
sign = 1
if study.direction == StudyDirection.MAXIMIZE:
sign = -1
states: Tuple[TrialState, ...]
if constant_liar:
states = (TrialState.COMPLETE, TrialState.PRUNED, TrialState.RUNNING)
else:
states = (TrialState.COMPLETE, TrialState.PRUNED)
scores = []
values: Dict[str, List[Optional[float]]] = {param_name: [] for param_name in param_names}
for trial in study.get_trials(deepcopy=False, states=states):
# If ``group`` = True, there may be trials that are not included in each subspace.
# Such trials should be ignored here.
if any([param_name not in trial.params for param_name in param_names]):
continue
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.value is None:
continue
score = (-float("inf"), sign * trial.value)
elif trial.state is TrialState.PRUNED:
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, float("inf"))
else:
score = (-step, sign * intermediate_value)
else:
score = (float("inf"), 0.0)
elif trial.state is TrialState.RUNNING:
assert constant_liar
score = (-float("inf"), sign * float("inf"))
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
assert param_name in trial.params
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(trial.params[param_name])
values[param_name].append(param_value)
return values, scores
def crossover(
self,
parents_params: np.ndarray,
rng: np.random.RandomState,
study: Study,
search_space_bounds: np.ndarray,
) -> np.ndarray:
# https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.422.952&rep=rep1&type=pdf
# Section 3.2 Crossover Schemes (vSBX)
if self._eta is None:
eta = 20.0 if study._is_multi_objective() else 2.0
else:
eta = self._eta
us = rng.rand(len(search_space_bounds))
beta_1 = np.power(1 / 2 * us, 1 / (eta + 1))
beta_2 = np.power(1 / 2 * (1 - us), 1 / (eta + 1))
mask = us > 0.5
c1 = 0.5 * ((1 + beta_1) * parents_params[0] +
(1 - beta_1) * parents_params[1])
c1[mask] = (0.5 * ((1 - beta_1) * parents_params[0] +
(1 + beta_1) * parents_params[1])[mask])
c2 = 0.5 * ((3 - beta_2) * parents_params[0] -
(1 - beta_2) * parents_params[1])
c2[mask] = (0.5 * (-(1 - beta_2) * parents_params[0] +
(3 - beta_2) * parents_params[1])[mask])
# vSBX applies crossover with establishment 0.5, and with probability 0.5,
# the gene of the parent individual is the gene of the child individual.
# The original SBX creates two child individuals,
# but optuna's implementation creates only one child individual.
# Therefore, when there is no crossover,
# the gene is selected with equal probability from the parent individuals x1 and x2.
child_params_list = []
for c1_i, c2_i, x1_i, x2_i in zip(c1, c2, parents_params[0],
parents_params[1]):
if rng.rand() < 0.5:
if rng.rand() < 0.5:
child_params_list.append(c1_i)
else:
child_params_list.append(c2_i)
else:
if rng.rand() < 0.5:
child_params_list.append(x1_i)
else:
child_params_list.append(x2_i)
child_params = np.array(child_params_list)
return child_params
def get_intermediate_plot(study: Study,topx=10,num_trials_threshold=30,color_scale=None) -> "go.Figure":
layout = go.Layout(
title="Intermediate Values Plot",
xaxis={"title": "Step"},
yaxis={"title": "Intermediate Value"},
)
#this value will determine which trials we want
df=study.trials_dataframe()
v=df.sort_values(by=["value"],ascending=True).reset_index().iloc[topx]["value"]
target_state = [TrialState.PRUNED, TrialState.COMPLETE, TrialState.RUNNING]
trials = [trial for trial in study.trials if (trial.state in target_state)
and (trial.value and trial.value<v) and len(trial.intermediate_values)<num_trials_threshold]
if len(trials) == 0:
_logger.warning("Study instance does not contain trials.")
return go.Figure(data=[], layout=layout)
traces = []
for i,trial in enumerate(trials):
if trial.intermediate_values:
sorted_intermediate_values = sorted(trial.intermediate_values.items())
trace = go.Scatter(
x=tuple((x for x, _ in sorted_intermediate_values)),
y=tuple((y for _, y in sorted_intermediate_values)),
mode="lines+markers",
marker={"maxdisplayed": 10},
marker_symbol=i if i<4 else i*2,
marker_size=10,
marker_color=color_scale[i],
name="Trial{}".format(trial.number),
)
traces.append(trace)
if not traces:
_logger.warning(
"You need to set up the pruning feature to utilize `plot_intermediate_values()`"
)
return go.Figure(data=[], layout=layout)
figure = go.Figure(data=traces, layout=layout,)
return figure
def _get_trials(self, study: Study) -> List[FrozenTrial]:
complete_trials = []
for t in study.get_trials(deepcopy=False):
if t.state == TrialState.COMPLETE:
complete_trials.append(t)
elif (t.state == TrialState.PRUNED
and len(t.intermediate_values) > 0
and self._consider_pruned_trials):
_, value = max(t.intermediate_values.items())
if value is None:
continue
# We rewrite the value of the trial `t` for sampling, so we need a deepcopy.
copied_t = copy.deepcopy(t)
copied_t.value = value
complete_trials.append(copied_t)
return complete_trials
def _get_intermediate_plot_info(study: Study) -> _IntermediatePlotInfo:
trials = study.get_trials(deepcopy=False,
states=(TrialState.PRUNED, TrialState.COMPLETE,
TrialState.RUNNING))
trial_infos = [
_TrialInfo(trial.number, sorted(trial.intermediate_values.items()))
for trial in trials if len(trial.intermediate_values) > 0
]
if len(trials) == 0:
_logger.warning("Study instance does not contain trials.")
elif len(trial_infos) == 0:
_logger.warning(
"You need to set up the pruning feature to utilize `plot_intermediate_values()`"
)
return _IntermediatePlotInfo(trial_infos)
def _get_multivariate_observation_pairs(
study: Study, param_names: List[str]
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, float]]]:
sign = 1
if study.direction == StudyDirection.MAXIMIZE:
sign = -1
scores = []
values: Dict[str, List[Optional[float]]] = {
param_name: []
for param_name in param_names
}
for trial in study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE,
TrialState.PRUNED)):
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.value is None:
continue
score = (-float("inf"), sign * trial.value)
elif trial.state is TrialState.PRUNED:
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(
trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, float("inf"))
else:
score = (-step, sign * intermediate_value)
else:
score = (float("inf"), 0.0)
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
assert param_name in trial.params
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(
trial.params[param_name])
values[param_name].append(param_value)
return values, scores
def sample_independent(
self,
study: Study,
trial: FrozenTrial,
param_name: str,
param_distribution: BaseDistribution,
) -> Any:
values, scores = _get_observation_pairs(study, [param_name],
self._multivariate,
self._constant_liar)
n = len(scores)
if n < self._n_startup_trials:
return self._random_sampler.sample_independent(
study, trial, param_name, param_distribution)
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n))
below = _build_observation_dict(values, indices_below)
above = _build_observation_dict(values, indices_above)
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
values, scores, indices_below)
mpe_below = _ParzenEstimator(
below,
{param_name: param_distribution},
self._parzen_estimator_parameters,
weights_below,
)
else:
mpe_below = _ParzenEstimator(below,
{param_name: param_distribution},
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, {param_name: param_distribution},
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
return param_distribution.to_external_repr(ret[param_name])
def _sample_relative(
self, study: Study, trial: FrozenTrial,
search_space: Dict[str, BaseDistribution]) -> Dict[str, Any]:
if search_space == {}:
return {}
param_names = list(search_space.keys())
values, scores = _get_observation_pairs(study, param_names,
self._multivariate,
self._constant_liar)
# If the number of samples is insufficient, we run random trial.
n = len(scores)
if n < self._n_startup_trials:
return {}
# We divide data into below and above.
indices_below, indices_above = _split_observation_pairs(
scores, self._gamma(n))
below = _build_observation_dict(values, indices_below)
above = _build_observation_dict(values, indices_above)
# We then sample by maximizing log likelihood ratio.
if study._is_multi_objective():
weights_below = _calculate_weights_below_for_multi_objective(
values, scores, indices_below)
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters,
weights_below)
else:
mpe_below = _ParzenEstimator(below, search_space,
self._parzen_estimator_parameters)
mpe_above = _ParzenEstimator(above, search_space,
self._parzen_estimator_parameters)
samples_below = mpe_below.sample(self._rng, self._n_ei_candidates)
log_likelihoods_below = mpe_below.log_pdf(samples_below)
log_likelihoods_above = mpe_above.log_pdf(samples_below)
ret = TPESampler._compare(samples_below, log_likelihoods_below,
log_likelihoods_above)
for param_name, dist in search_space.items():
ret[param_name] = dist.to_external_repr(ret[param_name])
return ret
def infer_relative_search_space(
self, study: Study,
trial: FrozenTrial) -> Dict[str, BaseDistribution]:
if self._initial_search_space is not None:
return self._initial_search_space
past_trials = study.get_trials(deepcopy=False,
states=_SUGGESTED_STATES)
# The initial trial is sampled by the independent sampler.
if len(past_trials) == 0:
return {}
# If an initial trial was already made,
# construct search_space of this sampler from the initial trial.
first_trial = min(past_trials, key=lambda t: t.number)
self._initial_search_space = self._infer_initial_search_space(
first_trial)
return self._initial_search_space
def sample_independent(
self,
study: Study,
trial: FrozenTrial,
param_name: str,
param_distribution: BaseDistribution,
) -> float:
self._raise_error_if_multi_objective(study)
if self._warn_independent_sampling:
complete_trials = study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE, ))
if len(complete_trials) >= self._n_startup_trials:
self._log_independent_sampling(trial, param_name)
return self._independent_sampler.sample_independent(
study, trial, param_name, param_distribution)
def infer_relative_search_space(
self, study: Study,
trial: FrozenTrial) -> Dict[str, BaseDistribution]:
if not self._multivariate:
return {}
search_space: Dict[str, BaseDistribution] = {}
for name, distribution in self._search_space.calculate(study).items():
if not isinstance(distribution, _DISTRIBUTION_CLASSES):
if self._warn_independent_sampling:
complete_trials = study.get_trials(deepcopy=False)
if len(complete_trials) >= self._n_startup_trials:
self._log_independent_sampling(trial, name)
continue
search_space[name] = distribution
return search_space
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params=params)
if params is None:
params = list(distributions.keys())
assert params is not None
if len(params) == 0:
return OrderedDict()
trials: List[FrozenTrial] = _get_filtered_trials(study, params=params, target=target)
trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False)
trans_params: np.ndarray = _get_trans_params(trials, trans)
target_values: np.ndarray = _get_target_values(trials, target)
forest = self._forest
forest.fit(X=trans_params, y=target_values)
# Create Tree Explainer object that can calculate shap values.
explainer = TreeExplainer(forest)
# Generate SHAP values for the parameters during the trials.
feature_shap_values: np.ndarray = explainer.shap_values(trans_params)
param_shap_values = np.zeros((len(trials), len(params)))
np.add.at(param_shap_values.T, trans.encoded_column_to_column, feature_shap_values.T)
# Calculate the mean absolute SHAP value for each parameter.
# List of tuples ("feature_name": mean_abs_shap_value).
mean_abs_shap_values = np.abs(param_shap_values).mean(axis=0)
return _sort_dict_by_importance(_param_importances_to_dict(params, mean_abs_shap_values))
def calculate(self, study: Study) -> _SearchSpaceGroup:
if self._study_id is None:
self._study_id = study._study_id
else:
# Note that the check below is meaningless when `InMemoryStorage` is used
# because `InMemoryStorage.create_new_study` always returns the same study ID.
if self._study_id != study._study_id:
raise ValueError("`_GroupDecomposedSearchSpace` cannot handle multiple studies.")
states_of_interest: Tuple[TrialState, ...]
if self._include_pruned:
states_of_interest = (TrialState.COMPLETE, TrialState.PRUNED)
else:
states_of_interest = (TrialState.COMPLETE,)
for trial in study.get_trials(deepcopy=False, states=states_of_interest):
self._search_space.add_distributions(trial.distributions)
return copy.deepcopy(self._search_space)
def _get_distributions(
study: Study,
params: Optional[List[str]]) -> Dict[str, BaseDistribution]:
completed_trials = study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE, ))
_check_evaluate_args(completed_trials, params)
if params is None:
return intersection_search_space(study, ordered_dict=True)
# New temporary required to pass mypy. Seems like a bug.
params_not_none = params
assert params_not_none is not None
# Compute the search space based on the subset of trials containing all parameters.
distributions = None
for trial in completed_trials:
trial_distributions = trial.distributions
if not all(name in trial_distributions for name in params_not_none):
continue
if distributions is None:
distributions = dict(
filter(
lambda name_and_distribution: name_and_distribution[0] in
params_not_none,
trial_distributions.items(),
))
continue
if any(trial_distributions[name] != distribution
for name, distribution in distributions.items()):
raise ValueError(
"Parameters importances cannot be assessed with dynamic search spaces if "
"parameters are specified. Specified parameters: {}.".format(
params))
assert distributions is not None # Required to pass mypy.
distributions = OrderedDict(
sorted(distributions.items(),
key=lambda name_and_distribution: name_and_distribution[0]))
return distributions
def _get_importances_info(
study: Study,
evaluator: Optional[BaseImportanceEvaluator],
params: Optional[List[str]],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> _ImportancesInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
logger.warning("Study instance does not contain completed trials.")
return _ImportancesInfo(
importance_values=[],
param_names=[],
importance_labels=[],
target_name=target_name,
)
importances = optuna.importance.get_param_importances(study,
evaluator=evaluator,
params=params,
target=target)
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
importance_labels = [
f"{val:.2f}" if val >= 0.01 else "<0.01" for val in importance_values
]
return _ImportancesInfo(
importance_values=importance_values,
param_names=param_names,
importance_labels=importance_labels,
target_name=target_name,
)
def _get_slice_plot_info(
study: Study,
params: Optional[List[str]],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> _SlicePlotInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return _SlicePlotInfo(target_name, [])
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
f"Parameter {input_p_name} does not exist in your study.")
sorted_params = sorted(set(params))
return _SlicePlotInfo(
target_name=target_name,
subplots=[
_get_slice_subplot_info(
trials=trials,
param=param,
target=target,
log_scale=_is_log_scale(trials, param),
numerical=_is_numerical(trials, param),
) for param in sorted_params
],
)
def create_log_file(study: Study, log_file: str):
"""
Create Log File
===============
Allows to create a log file containing a pandas `DataFrame` of trials in the `Study`.
Parameters
----------
study : optuna.study.Study
Set of `Trial` objects deriving from an Ask-and-Tell interface.
log_file : str
Name of file reporting the study results.
Returns
-------
None
"""
df = study.trials_dataframe(attrs=('number', 'params', 'value', 'state'))
with open(log_file, 'w') as file:
print(df.to_string(), file=file)
