def infer_relative_search_space(
self, study: Study, trial: FrozenTrial
) -> Dict[str, BaseDistribution]:
if not self._multivariate:
return {}
n_complete_trials = len(study.get_trials(deepcopy=False))
search_space: Dict[str, BaseDistribution] = {}
if self._group:
assert self._group_decomposed_search_space is not None
self._search_space_group = self._group_decomposed_search_space.calculate(study)
for sub_space in self._search_space_group.search_spaces:
for name, distribution in sub_space.items():
if not isinstance(distribution, _DISTRIBUTION_CLASSES):
self._log_independent_sampling(n_complete_trials, trial, name)
continue
search_space[name] = distribution
return search_space
for name, distribution in self._search_space.calculate(study).items():
if not isinstance(distribution, _DISTRIBUTION_CLASSES):
self._log_independent_sampling(n_complete_trials, trial, name)
continue
search_space[name] = distribution
return search_space
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
trials = []
for trial in _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions,
transform_log=False,
transform_step=False)
n_trials = len(trials)
self._trans_params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
self._trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
self._trans_params[trial_idx] = trans.transform(trial.params)
self._trans_values[
trial_idx] = trial.value if target is None else target(trial)
encoded_column_to_column = trans.encoded_column_to_column
if self._trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
forest = self._forest
forest.fit(self._trans_params, self._trans_values)
feature_importances = forest.feature_importances_
feature_importances_reduced = numpy.zeros(len(distributions))
numpy.add.at(feature_importances_reduced, encoded_column_to_column,
feature_importances)
param_importances = OrderedDict()
self._param_names = list(distributions.keys())
for i in feature_importances_reduced.argsort()[::-1]:
param_importances[
self._param_names[i]] = feature_importances_reduced[i].item()
return param_importances
def _get_non_pareto_front_trials(
study: Study, pareto_trials: List[FrozenTrial]) -> List[FrozenTrial]:
non_pareto_trials = []
for trial in study.get_trials():
if trial.state == TrialState.COMPLETE and trial not in pareto_trials:
non_pareto_trials.append(trial)
return non_pareto_trials
def _get_observation_pairs(
study: Study, param_name: str
) -> Tuple[List[Optional[float]], List[Tuple[float, float]]]:
"""Get observation pairs from the study.
This function collects observation pairs from the complete or pruned trials of the study.
The values for trials that don't contain the parameter named ``param_name`` are set to None.
An observation pair fundamentally consists of a parameter value and an objective value.
However, due to the pruning mechanism of Optuna, final objective values are not always
available. Therefore, this function uses intermediate values in addition to the final
ones, and reports the value with its step count as ``(-step, value)``.
Consequently, the structure of the observation pair is as follows:
``(param_value, (-step, value))``.
The second element of an observation pair is used to rank observations in
``_split_observation_pairs`` method (i.e., observations are sorted lexicographically by
``(-step, value)``).
"""
sign = 1
if study.direction == StudyDirection.MAXIMIZE:
sign = -1
values = []
scores = []
for trial in study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE,
TrialState.PRUNED)):
if trial.state is TrialState.COMPLETE:
if trial.value is None:
continue
score = (-float("inf"), sign * trial.value)
elif trial.state is TrialState.PRUNED:
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(
trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, float("inf"))
else:
score = (-step, sign * intermediate_value)
else:
score = (float("inf"), 0.0)
else:
assert False
param_value: Optional[float] = None
if param_name in trial.params:
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(
trial.params[param_name])
values.append(param_value)
scores.append(score)
return values, scores
def _get_filtered_trials(
study: Study, params: Collection[str], target: Optional[Callable[[FrozenTrial], float]]
) -> List[FrozenTrial]:
trials = study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,))
return [
trial
for trial in trials
if set(params) <= set(trial.params)
and numpy.isfinite(target(trial) if target is not None else cast(float, trial.value))
]
def _get_contour_info(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> _ContourInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
sorted_params = []
elif params is None:
sorted_params = sorted(all_params)
else:
if len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
"Parameter {} does not exist in your study.".format(
input_p_name))
sorted_params = sorted(set(params))
sub_plot_infos: List[List[_SubContourInfo]]
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plot_info = _get_contour_subplot_info(trials, x_param, y_param,
target)
sub_plot_infos = [[sub_plot_info]]
else:
sub_plot_infos = []
for i, y_param in enumerate(sorted_params):
sub_plot_infos.append([])
for x_param in sorted_params:
sub_plot_info = _get_contour_subplot_info(
trials, x_param, y_param, target)
sub_plot_infos[i].append(sub_plot_info)
reverse_scale = _is_reverse_scale(study, target)
return _ContourInfo(
sorted_params=sorted_params,
sub_plot_infos=sub_plot_infos,
reverse_scale=reverse_scale,
target_name=target_name,
)
def _get_multivariate_observation_pairs(
study: Study,
param_names: List[str],
constant_liar: bool = False, # TODO(hvy): Remove default value and fix unit tests.
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, float]]]:
sign = 1
if study.direction == StudyDirection.MAXIMIZE:
sign = -1
states: Tuple[TrialState, ...]
if constant_liar:
states = (TrialState.COMPLETE, TrialState.PRUNED, TrialState.RUNNING)
else:
states = (TrialState.COMPLETE, TrialState.PRUNED)
scores = []
values: Dict[str, List[Optional[float]]] = {param_name: [] for param_name in param_names}
for trial in study.get_trials(deepcopy=False, states=states):
# If ``group`` = True, there may be trials that are not included in each subspace.
# Such trials should be ignored here.
if any([param_name not in trial.params for param_name in param_names]):
continue
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.value is None:
continue
score = (-float("inf"), sign * trial.value)
elif trial.state is TrialState.PRUNED:
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, float("inf"))
else:
score = (-step, sign * intermediate_value)
else:
score = (float("inf"), 0.0)
elif trial.state is TrialState.RUNNING:
assert constant_liar
score = (-float("inf"), sign * float("inf"))
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
assert param_name in trial.params
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(trial.params[param_name])
values[param_name].append(param_value)
return values, scores
def _get_trials(self, study: Study) -> List[FrozenTrial]:
complete_trials = []
for t in study.get_trials(deepcopy=False):
if t.state == TrialState.COMPLETE:
complete_trials.append(t)
elif (t.state == TrialState.PRUNED
and len(t.intermediate_values) > 0
and self._consider_pruned_trials):
_, value = max(t.intermediate_values.items())
if value is None:
continue
# We rewrite the value of the trial `t` for sampling, so we need a deepcopy.
copied_t = copy.deepcopy(t)
copied_t.value = value
complete_trials.append(copied_t)
return complete_trials
def _get_intermediate_plot_info(study: Study) -> _IntermediatePlotInfo:
trials = study.get_trials(deepcopy=False,
states=(TrialState.PRUNED, TrialState.COMPLETE,
TrialState.RUNNING))
trial_infos = [
_TrialInfo(trial.number, sorted(trial.intermediate_values.items()))
for trial in trials if len(trial.intermediate_values) > 0
]
if len(trials) == 0:
_logger.warning("Study instance does not contain trials.")
elif len(trial_infos) == 0:
_logger.warning(
"You need to set up the pruning feature to utilize `plot_intermediate_values()`"
)
return _IntermediatePlotInfo(trial_infos)
def _get_multivariate_observation_pairs(
study: Study, param_names: List[str]
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, float]]]:
sign = 1
if study.direction == StudyDirection.MAXIMIZE:
sign = -1
scores = []
values: Dict[str, List[Optional[float]]] = {
param_name: []
for param_name in param_names
}
for trial in study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE,
TrialState.PRUNED)):
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.value is None:
continue
score = (-float("inf"), sign * trial.value)
elif trial.state is TrialState.PRUNED:
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(
trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, float("inf"))
else:
score = (-step, sign * intermediate_value)
else:
score = (float("inf"), 0.0)
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
assert param_name in trial.params
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(
trial.params[param_name])
values[param_name].append(param_value)
return values, scores
def infer_relative_search_space(
self, study: Study,
trial: FrozenTrial) -> Dict[str, BaseDistribution]:
if self._initial_search_space is not None:
return self._initial_search_space
past_trials = study.get_trials(deepcopy=False,
states=_SUGGESTED_STATES)
# The initial trial is sampled by the independent sampler.
if len(past_trials) == 0:
return {}
# If an initial trial was already made,
# construct search_space of this sampler from the initial trial.
first_trial = min(past_trials, key=lambda t: t.number)
self._initial_search_space = self._infer_initial_search_space(
first_trial)
return self._initial_search_space
def sample_independent(
self,
study: Study,
trial: FrozenTrial,
param_name: str,
param_distribution: BaseDistribution,
) -> float:
self._raise_error_if_multi_objective(study)
if self._warn_independent_sampling:
complete_trials = study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE, ))
if len(complete_trials) >= self._n_startup_trials:
self._log_independent_sampling(trial, param_name)
return self._independent_sampler.sample_independent(
study, trial, param_name, param_distribution)
def infer_relative_search_space(
self, study: Study,
trial: FrozenTrial) -> Dict[str, BaseDistribution]:
if not self._multivariate:
return {}
search_space: Dict[str, BaseDistribution] = {}
for name, distribution in self._search_space.calculate(study).items():
if not isinstance(distribution, _DISTRIBUTION_CLASSES):
if self._warn_independent_sampling:
complete_trials = study.get_trials(deepcopy=False)
if len(complete_trials) >= self._n_startup_trials:
self._log_independent_sampling(trial, name)
continue
search_space[name] = distribution
return search_space
def calculate(self, study: Study) -> _SearchSpaceGroup:
if self._study_id is None:
self._study_id = study._study_id
else:
# Note that the check below is meaningless when `InMemoryStorage` is used
# because `InMemoryStorage.create_new_study` always returns the same study ID.
if self._study_id != study._study_id:
raise ValueError("`_GroupDecomposedSearchSpace` cannot handle multiple studies.")
states_of_interest: Tuple[TrialState, ...]
if self._include_pruned:
states_of_interest = (TrialState.COMPLETE, TrialState.PRUNED)
else:
states_of_interest = (TrialState.COMPLETE,)
for trial in study.get_trials(deepcopy=False, states=states_of_interest):
self._search_space.add_distributions(trial.distributions)
return copy.deepcopy(self._search_space)
def _get_distributions(
study: Study,
params: Optional[List[str]]) -> Dict[str, BaseDistribution]:
completed_trials = study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE, ))
_check_evaluate_args(completed_trials, params)
if params is None:
return intersection_search_space(study, ordered_dict=True)
# New temporary required to pass mypy. Seems like a bug.
params_not_none = params
assert params_not_none is not None
# Compute the search space based on the subset of trials containing all parameters.
distributions = None
for trial in completed_trials:
trial_distributions = trial.distributions
if not all(name in trial_distributions for name in params_not_none):
continue
if distributions is None:
distributions = dict(
filter(
lambda name_and_distribution: name_and_distribution[0] in
params_not_none,
trial_distributions.items(),
))
continue
if any(trial_distributions[name] != distribution
for name, distribution in distributions.items()):
raise ValueError(
"Parameters importances cannot be assessed with dynamic search spaces if "
"parameters are specified. Specified parameters: {}.".format(
params))
assert distributions is not None # Required to pass mypy.
distributions = OrderedDict(
sorted(distributions.items(),
key=lambda name_and_distribution: name_and_distribution[0]))
return distributions
def _get_importances_info(
study: Study,
evaluator: Optional[BaseImportanceEvaluator],
params: Optional[List[str]],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> _ImportancesInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
logger.warning("Study instance does not contain completed trials.")
return _ImportancesInfo(
importance_values=[],
param_names=[],
importance_labels=[],
target_name=target_name,
)
importances = optuna.importance.get_param_importances(study,
evaluator=evaluator,
params=params,
target=target)
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
importance_labels = [
f"{val:.2f}" if val >= 0.01 else "<0.01" for val in importance_values
]
return _ImportancesInfo(
importance_values=importance_values,
param_names=param_names,
importance_labels=importance_labels,
target_name=target_name,
)
def _get_slice_plot_info(
study: Study,
params: Optional[List[str]],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> _SlicePlotInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return _SlicePlotInfo(target_name, [])
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
f"Parameter {input_p_name} does not exist in your study.")
sorted_params = sorted(set(params))
return _SlicePlotInfo(
target_name=target_name,
subplots=[
_get_slice_subplot_info(
trials=trials,
param=param,
target=target,
log_scale=_is_log_scale(trials, param),
numerical=_is_numerical(trials, param),
) for param in sorted_params
],
)
def sample_relative(
self, study: Study, trial: FrozenTrial,
search_space: Dict[str, BaseDistribution]) -> Dict[str, float]:
self._raise_error_if_multi_objective(study)
if len(search_space) == 0:
return {}
if len(search_space) == 1:
_logger.info(
"`PyCmaSampler` does not support optimization of 1-D search space. "
"`{}` is used instead of `PyCmaSampler`.".format(
self._independent_sampler.__class__.__name__))
self._warn_independent_sampling = False
return {}
complete_trials = study.get_trials(deepcopy=False,
states=(TrialState.COMPLETE, ))
if len(complete_trials) < self._n_startup_trials:
return {}
if self._x0 is None:
self._x0 = self._initialize_x0(search_space)
if self._sigma0 is None:
sigma0 = self._initialize_sigma0(search_space)
else:
sigma0 = self._sigma0
# Avoid ZeroDivisionError in cma.CMAEvolutionStrategy.
sigma0 = max(sigma0, _EPS)
optimizer = _Optimizer(search_space, self._x0, sigma0, self._cma_stds,
self._cma_opts)
trials = study.trials
last_told_trial_number = optimizer.tell(trials, study.direction)
return optimizer.ask(trials, last_told_trial_number)
def sample_relative(
self,
study: Study,
trial: FrozenTrial,
search_space: Dict[str, BaseDistribution],
) -> Dict[str, Any]:
assert isinstance(search_space, OrderedDict)
if len(search_space) == 0:
return {}
trials = [
t for t in study.get_trials(deepcopy=False)
if t.state == TrialState.COMPLETE
]
n_trials = len(trials)
if n_trials < self._n_startup_trials:
return {}
trans = _SearchSpaceTransform(search_space)
n_objectives = len(study.directions)
values = numpy.empty((n_trials, n_objectives), dtype=numpy.float64)
params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
con = None
bounds = trans.bounds
for trial_idx, trial in enumerate(trials):
params[trial_idx] = trans.transform(trial.params)
assert len(study.directions) == len(trial.values)
for obj_idx, (direction, value) in enumerate(
zip(study.directions, trial.values)):
assert value is not None
if direction == StudyDirection.MINIMIZE: # BoTorch always assumes maximization.
value *= -1
values[trial_idx, obj_idx] = value
if self._constraints_func is not None:
constraints = study._storage.get_trial_system_attrs(
trial._trial_id).get("botorch:constraints")
if constraints is not None:
n_constraints = len(constraints)
if con is None:
con = numpy.full((n_trials, n_constraints),
numpy.nan,
dtype=numpy.float64)
elif n_constraints != con.shape[1]:
raise RuntimeError(
f"Expected {con.shape[1]} constraints but received {n_constraints}."
)
con[trial_idx] = constraints
if self._constraints_func is not None:
if con is None:
warnings.warn(
"`constraints_func` was given but no call to it correctly computed "
"constraints. Constraints passed to `candidates_func` will be `None`."
)
elif numpy.isnan(con).any():
warnings.warn(
"`constraints_func` was given but some calls to it did not correctly compute "
"constraints. Constraints passed to `candidates_func` will contain NaN."
)
values = torch.from_numpy(values)
params = torch.from_numpy(params)
if con is not None:
con = torch.from_numpy(con)
bounds = torch.from_numpy(bounds)
if con is not None:
if con.dim() == 1:
con.unsqueeze_(-1)
bounds.transpose_(0, 1)
if self._candidates_func is None:
self._candidates_func = _get_default_candidates_func(
n_objectives=n_objectives)
candidates = self._candidates_func(params, values, con, bounds)
if not isinstance(candidates, torch.Tensor):
raise TypeError("Candidates must be a torch.Tensor.")
if candidates.dim() == 2:
if candidates.size(0) != 1:
raise ValueError(
"Candidates batch optimization is not supported and the first dimension must "
"have size 1 if candidates is a two-dimensional tensor. Actual: "
f"{candidates.size()}.")
# Batch size is one. Get rid of the batch dimension.
candidates = candidates.squeeze(0)
if candidates.dim() != 1:
raise ValueError("Candidates must be one or two-dimensional.")
if candidates.size(0) != bounds.size(1):
raise ValueError(
"Candidates size must match with the given bounds. Actual candidates: "
f"{candidates.size(0)}, bounds: {bounds.size(1)}.")
candidates = candidates.numpy()
params = trans.untransform(candidates)
return params
def _collect_parent_population(
self, study: Study) -> Tuple[int, List[FrozenTrial]]:
trials = study.get_trials(deepcopy=False)
generation_to_runnings = defaultdict(list)
generation_to_population = defaultdict(list)
for trial in trials:
if _GENERATION_KEY not in trial.system_attrs:
continue
generation = trial.system_attrs[_GENERATION_KEY]
if trial.state != optuna.trial.TrialState.COMPLETE:
if trial.state == optuna.trial.TrialState.RUNNING:
generation_to_runnings[generation].append(trial)
continue
# Do not use trials whose states are not COMPLETE, or `constraint` will be unavailable.
generation_to_population[generation].append(trial)
hasher = hashlib.sha256()
parent_population: List[FrozenTrial] = []
parent_generation = -1
while True:
generation = parent_generation + 1
population = generation_to_population[generation]
# Under multi-worker settings, the population size might become larger than
# `self._population_size`.
if len(population) < self._population_size:
break
# [NOTE]
# It's generally safe to assume that once the above condition is satisfied,
# there are no additional individuals added to the generation (i.e., the members of
# the generation have been fixed).
# If the number of parallel workers is huge, this assumption can be broken, but
# this is a very rare case and doesn't significantly impact optimization performance.
# So we can ignore the case.
# The cache key is calculated based on the key of the previous generation and
# the remaining running trials in the current population.
# If there are no running trials, the new cache key becomes exactly the same as
# the previous one, and the cached content will be overwritten. This allows us to
# skip redundant cache key calculations when this method is called for the subsequent
# trials.
for trial in generation_to_runnings[generation]:
hasher.update(bytes(str(trial.number), "utf-8"))
cache_key = "{}:{}".format(_POPULATION_CACHE_KEY_PREFIX,
hasher.hexdigest())
cached_generation, cached_population_numbers = study.system_attrs.get(
cache_key, (-1, []))
if cached_generation >= generation:
generation = cached_generation
population = [trials[n] for n in cached_population_numbers]
else:
population.extend(parent_population)
population = self._select_elite_population(study, population)
# To reduce the number of system attribute entries,
# we cache the population information only if there are no running trials
# (i.e., the information of the population has been fixed).
# Usually, if there are no too delayed running trials, the single entry
# will be used.
if len(generation_to_runnings[generation]) == 0:
population_numbers = [t.number for t in population]
study.set_system_attr(cache_key,
(generation, population_numbers))
parent_generation = generation
parent_population = population
return parent_generation, parent_population
def calculate(self,
study: Study,
ordered_dict: bool = False) -> Dict[str, BaseDistribution]:
"""Returns the intersection search space of the :class:`~optuna.study.Study`.
Args:
study:
A study with completed trials.
ordered_dict:
A boolean flag determining the return type.
If :obj:`False`, the returned object will be a :obj:`dict`.
If :obj:`True`, the returned object will be an :obj:`collections.OrderedDict`
sorted by keys, i.e. parameter names.
Returns:
A dictionary containing the parameter names and parameter's distributions.
Raises:
ValueError:
If different studies are passed into this method.
"""
if self._study_id is None:
self._study_id = study._study_id
else:
# Note that the check below is meaningless when `InMemoryStorage` is used
# because `InMemoryStorage.create_new_study` always returns the same study ID.
if self._study_id != study._study_id:
raise ValueError(
"`IntersectionSearchSpace` cannot handle multiple studies."
)
states_of_interest = [optuna.trial.TrialState.COMPLETE]
if self._include_pruned:
states_of_interest.append(optuna.trial.TrialState.PRUNED)
next_cursor = self._cursor
for trial in reversed(study.get_trials(deepcopy=False)):
if self._cursor > trial.number:
break
if not trial.state.is_finished():
next_cursor = trial.number
if trial.state not in states_of_interest:
continue
if self._search_space is None:
self._search_space = copy.copy(trial.distributions)
continue
delete_list = []
for param_name, param_distribution in self._search_space.items():
if param_name not in trial.distributions:
delete_list.append(param_name)
elif trial.distributions[param_name] != param_distribution:
delete_list.append(param_name)
for param_name in delete_list:
del self._search_space[param_name]
self._cursor = next_cursor
search_space = self._search_space or {}
if ordered_dict:
search_space = OrderedDict(
sorted(search_space.items(), key=lambda x: x[0]))
return copy.deepcopy(search_space)
def _get_contour_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(title="Contour Plot")
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return go.Figure(data=[], layout=layout)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
elif len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
return go.Figure(data=[], layout=layout)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
"Parameter {} does not exist in your study.".format(
input_p_name))
sorted_params = sorted(set(params))
padding_ratio = 0.05
param_values_range = {}
for p_name in sorted_params:
values = _get_param_values(trials, p_name)
min_value = min(values)
max_value = max(values)
if _is_log_scale(trials, p_name):
padding = (math.log10(max_value) -
math.log10(min_value)) * padding_ratio
min_value = math.pow(10, math.log10(min_value) - padding)
max_value = math.pow(10, math.log10(max_value) + padding)
elif _is_numerical(trials, p_name):
padding = (max_value - min_value) * padding_ratio
min_value = min_value - padding
max_value = max_value + padding
else:
# Plotly>=4.12.0 draws contours using the indices of categorical variables instead of
# raw values and the range should be updated based on the cardinality of categorical
# variables. See https://github.com/optuna/optuna/issues/1967.
if version.parse(plotly.__version__) >= version.parse("4.12.0"):
span = len(set(values)) - 1
padding = span * padding_ratio
min_value = -padding
max_value = span + padding
param_values_range[p_name] = (min_value, max_value)
reverse_scale = _is_reverse_scale(study, target)
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plots = _generate_contour_subplot(trials, x_param, y_param,
reverse_scale,
param_values_range, target,
target_name)
figure = go.Figure(data=sub_plots, layout=layout)
figure.update_xaxes(title_text=x_param,
range=param_values_range[x_param])
figure.update_yaxes(title_text=y_param,
range=param_values_range[y_param])
if not _is_numerical(trials, x_param):
figure.update_xaxes(type="category")
if not _is_numerical(trials, y_param):
figure.update_yaxes(type="category")
if _is_log_scale(trials, x_param):
log_range = [math.log10(p) for p in param_values_range[x_param]]
figure.update_xaxes(range=log_range, type="log")
if _is_log_scale(trials, y_param):
log_range = [math.log10(p) for p in param_values_range[y_param]]
figure.update_yaxes(range=log_range, type="log")
else:
figure = make_subplots(rows=len(sorted_params),
cols=len(sorted_params),
shared_xaxes=True,
shared_yaxes=True)
figure.update_layout(layout)
showscale = True # showscale option only needs to be specified once
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
if x_param == y_param:
figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1)
else:
sub_plots = _generate_contour_subplot(
trials,
x_param,
y_param,
reverse_scale,
param_values_range,
target,
target_name,
)
contour = sub_plots[0]
scatter = sub_plots[1]
contour.update(
showscale=showscale) # showscale's default is True
if showscale:
showscale = False
figure.add_trace(contour, row=y_i + 1, col=x_i + 1)
figure.add_trace(scatter, row=y_i + 1, col=x_i + 1)
figure.update_xaxes(range=param_values_range[x_param],
row=y_i + 1,
col=x_i + 1)
figure.update_yaxes(range=param_values_range[y_param],
row=y_i + 1,
col=x_i + 1)
if not _is_numerical(trials, x_param):
figure.update_xaxes(type="category",
row=y_i + 1,
col=x_i + 1)
if not _is_numerical(trials, y_param):
figure.update_yaxes(type="category",
row=y_i + 1,
col=x_i + 1)
if _is_log_scale(trials, x_param):
log_range = [
math.log10(p) for p in param_values_range[x_param]
]
figure.update_xaxes(range=log_range,
type="log",
row=y_i + 1,
col=x_i + 1)
if _is_log_scale(trials, y_param):
log_range = [
math.log10(p) for p in param_values_range[y_param]
]
figure.update_yaxes(range=log_range,
type="log",
row=y_i + 1,
col=x_i + 1)
if x_i == 0:
figure.update_yaxes(title_text=y_param,
row=y_i + 1,
col=x_i + 1)
if y_i == len(sorted_params) - 1:
figure.update_xaxes(title_text=x_param,
row=y_i + 1,
col=x_i + 1)
return figure
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
# fANOVA does not support parameter distributions with a single value.
# However, there is no reason to calculate parameter importance in such case anyway,
# since it will always be 0 as the parameter is constant in the objective function.
zero_importances = {
name: 0.0
for name, dist in distributions.items() if dist.single()
}
distributions = {
name: dist
for name, dist in distributions.items() if not dist.single()
}
trials = []
for trial in _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions,
transform_log=False,
transform_step=False)
n_trials = len(trials)
trans_params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
trans_params[trial_idx] = trans.transform(trial.params)
trans_values[
trial_idx] = trial.value if target is None else target(trial)
trans_bounds = trans.bounds
column_to_encoded_columns = trans.column_to_encoded_columns
if trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
# Many (deep) copies of the search spaces are required during the tree traversal and using
# Optuna distributions will create a bottleneck.
# Therefore, search spaces (parameter distributions) are represented by a single
# `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical
# or not.
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=trans_values,
search_spaces=trans_bounds,
column_to_encoded_columns=column_to_encoded_columns,
)
importances = {}
for i, name in enumerate(distributions.keys()):
importance, _ = evaluator.get_importance((i, ))
importances[name] = importance
importances = {**importances, **zero_importances}
total_importance = sum(importances.values())
for name in importances:
importances[name] /= total_importance
sorted_importances = OrderedDict(
reversed(
sorted(
importances.items(),
key=lambda name_and_importance: name_and_importance[1])))
return sorted_importances
def _get_contour_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
# Calculate basic numbers for plotting.
trials = _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
_, ax = plt.subplots()
return ax
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
elif len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
_, ax = plt.subplots()
return ax
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError("Parameter {} does not exist in your study.".format(input_p_name))
sorted_params = sorted(set(params))
n_params = len(sorted_params)
plt.style.use("ggplot") # Use ggplot style sheet for similar outputs to plotly.
if n_params == 2:
# Set up the graph style.
fig, axs = plt.subplots()
axs.set_title("Contour Plot")
cmap = _set_cmap(study, target)
contour_point_num = 100
# Prepare data and draw contour plots.
if params:
x_param = params[0]
y_param = params[1]
else:
x_param = sorted_params[0]
y_param = sorted_params[1]
cs = _generate_contour_subplot(
trials, x_param, y_param, axs, cmap, contour_point_num, target
)
if isinstance(cs, ContourSet):
axcb = fig.colorbar(cs)
axcb.set_label(target_name)
else:
# Set up the graph style.
fig, axs = plt.subplots(n_params, n_params)
fig.suptitle("Contour Plot")
cmap = _set_cmap(study, target)
contour_point_num = 100
# Prepare data and draw contour plots.
cs_list = []
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
ax = axs[y_i, x_i]
cs = _generate_contour_subplot(
trials, x_param, y_param, ax, cmap, contour_point_num, target
)
if isinstance(cs, ContourSet):
cs_list.append(cs)
if cs_list:
axcb = fig.colorbar(cs_list[0], ax=axs)
axcb.set_label(target_name)
return axs
def plot_pareto_front(
study: Study,
*,
target_names: Optional[List[str]] = None,
include_dominated_trials: bool = True,
axis_order: Optional[List[int]] = None,
constraints_func: Optional[Callable[[FrozenTrial],
Sequence[float]]] = None,
) -> "go.Figure":
"""Plot the Pareto front of a study.
.. seealso::
Please refer to :ref:`multi_objective` for the tutorial of the Pareto front visualization.
Example:
The following code snippet shows how to plot the Pareto front of a study.
.. plotly::
import optuna
def objective(trial):
x = trial.suggest_float("x", 0, 5)
y = trial.suggest_float("y", 0, 3)
v0 = 4 * x ** 2 + 4 * y ** 2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
study = optuna.create_study(directions=["minimize", "minimize"])
study.optimize(objective, n_trials=50)
fig = optuna.visualization.plot_pareto_front(study)
fig.show()
Args:
study:
A :class:`~optuna.study.Study` object whose trials are plotted for their objective
values.
target_names:
Objective name list used as the axis titles. If :obj:`None` is specified,
"Objective {objective_index}" is used instead.
include_dominated_trials:
A flag to include all dominated trial's objective values.
axis_order:
A list of indices indicating the axis order. If :obj:`None` is specified,
default order is used.
constraints_func:
An optional function that computes the objective constraints. It must take a
:class:`~optuna.trial.FrozenTrial` and return the constraints. The return value must
be a sequence of :obj:`float` s. A value strictly larger than 0 means that a
constraint is violated. A value equal to or smaller than 0 is considered feasible.
This specification is the same as in, for example,
:class:`~optuna.integration.NSGAIISampler`.
If given, trials are classified into three categories: feasible and best, feasible but
non-best, and infeasible. Categories are shown in different colors. Here, whether a
trial is best (on Pareto front) or not is determined ignoring all infeasible trials.
Returns:
A :class:`plotly.graph_objs.Figure` object.
Raises:
:exc:`ValueError`:
If the number of objectives of ``study`` isn't 2 or 3.
"""
_imports.check()
n_dim = len(study.directions)
if n_dim not in (2, 3):
raise ValueError(
"`plot_pareto_front` function only supports 2 or 3 objective studies."
)
if target_names is None:
target_names = [f"Objective {i}" for i in range(n_dim)]
elif len(target_names) != n_dim:
raise ValueError(
f"The length of `target_names` is supposed to be {n_dim}.")
if constraints_func is not None:
feasible_trials = []
infeasible_trials = []
for trial in study.get_trials(states=(TrialState.COMPLETE, )):
if all(map(lambda x: x <= 0.0, constraints_func(trial))):
feasible_trials.append(trial)
else:
infeasible_trials.append(trial)
best_trials = _get_pareto_front_trials_by_trials(
feasible_trials, study.directions)
if include_dominated_trials:
non_best_trials = _get_non_pareto_front_trials(
feasible_trials, best_trials)
else:
non_best_trials = []
if len(best_trials) == 0:
_logger.warning(
"Your study does not have any completed and feasible trials.")
else:
best_trials = study.best_trials
if len(best_trials) == 0:
_logger.warning("Your study does not have any completed trials.")
if include_dominated_trials:
non_best_trials = _get_non_pareto_front_trials(
study.get_trials(deepcopy=False), best_trials)
else:
non_best_trials = []
infeasible_trials = []
if axis_order is None:
axis_order = list(range(n_dim))
else:
if len(axis_order) != n_dim:
raise ValueError(
f"Size of `axis_order` {axis_order}. Expect: {n_dim}, Actual: {len(axis_order)}."
)
if len(set(axis_order)) != n_dim:
raise ValueError(
f"Elements of given `axis_order` {axis_order} are not unique!."
)
if max(axis_order) > n_dim - 1:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} "
f"higher than {n_dim - 1}.")
if min(axis_order) < 0:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} "
"lower than 0.")
def _make_scatter_object(
trials: Sequence[FrozenTrial],
hovertemplate: str,
infeasible: bool = False,
dominated_trials: bool = False,
) -> Union["go.Scatter", "go.Scatter3d"]:
return _make_scatter_object_base(
n_dim,
trials,
axis_order, # type: ignore
include_dominated_trials,
hovertemplate=hovertemplate,
infeasible=infeasible,
dominated_trials=dominated_trials,
)
if constraints_func is None:
data = [
_make_scatter_object(
non_best_trials,
hovertemplate="%{text}<extra>Trial</extra>",
dominated_trials=True,
),
_make_scatter_object(
best_trials,
hovertemplate="%{text}<extra>Best Trial</extra>",
dominated_trials=False,
),
]
else:
data = [
_make_scatter_object(
infeasible_trials,
hovertemplate="%{text}<extra>Infeasible Trial</extra>",
infeasible=True,
),
_make_scatter_object(
non_best_trials,
hovertemplate="%{text}<extra>Feasible Trial</extra>",
dominated_trials=True,
),
_make_scatter_object(
best_trials,
hovertemplate="%{text}<extra>Best Trial</extra>",
dominated_trials=False,
),
]
if n_dim == 2:
layout = go.Layout(
title="Pareto-front Plot",
xaxis_title=target_names[axis_order[0]],
yaxis_title=target_names[axis_order[1]],
)
else:
layout = go.Layout(
title="Pareto-front Plot",
scene={
"xaxis_title": target_names[axis_order[0]],
"yaxis_title": target_names[axis_order[1]],
"zaxis_title": target_names[axis_order[2]],
},
)
return go.Figure(data=data, layout=layout)
def _get_observation_pairs(
study: Study,
param_names: List[str],
multivariate: bool,
constant_liar:
bool = False, # TODO(hvy): Remove default value and fix unit tests.
constraints_enabled: bool = False,
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, List[float]]],
Optional[List[float]], ]:
"""Get observation pairs from the study.
This function collects observation pairs from the complete or pruned trials of the study.
In addition, if ``constant_liar`` is :obj:`True`, the running trials are considered.
The values for trials that don't contain the parameter in the ``param_names`` are skipped.
An observation pair fundamentally consists of a parameter value and an objective value.
However, due to the pruning mechanism of Optuna, final objective values are not always
available. Therefore, this function uses intermediate values in addition to the final
ones, and reports the value with its step count as ``(-step, value)``.
Consequently, the structure of the observation pair is as follows:
``(param_value, (-step, value))``.
The second element of an observation pair is used to rank observations in
``_split_observation_pairs`` method (i.e., observations are sorted lexicographically by
``(-step, value)``).
When ``constraints_enabled`` is :obj:`True`, 1-dimensional violation values are returned
as the third element (:obj:`None` otherwise). Each value is a float of 0 or greater and a
trial is feasible if and only if its violation score is 0.
"""
if len(param_names) > 1:
assert multivariate
signs = []
for d in study.directions:
if d == StudyDirection.MINIMIZE:
signs.append(1)
else:
signs.append(-1)
states: Container[TrialState]
if constant_liar:
states = (TrialState.COMPLETE, TrialState.PRUNED, TrialState.RUNNING)
else:
states = (TrialState.COMPLETE, TrialState.PRUNED)
scores = []
values: Dict[str, List[Optional[float]]] = {
param_name: []
for param_name in param_names
}
violations: Optional[List[float]] = [] if constraints_enabled else None
for trial in study.get_trials(deepcopy=False, states=states):
# If ``multivariate`` = True and ``group`` = True, we ignore the trials that are not
# included in each subspace.
# If ``multivariate`` = False, we skip the check.
if multivariate and any(
[param_name not in trial.params for param_name in param_names]):
continue
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.values is None:
continue
score = (-float("inf"),
[sign * v for sign, v in zip(signs, trial.values)])
elif trial.state is TrialState.PRUNED:
if study._is_multi_objective():
continue
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(
trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, [float("inf")])
else:
score = (-step, [signs[0] * intermediate_value])
else:
score = (float("inf"), [0.0])
elif trial.state is TrialState.RUNNING:
if study._is_multi_objective():
continue
assert constant_liar
score = (-float("inf"), [signs[0] * float("inf")])
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
param_value: Optional[float]
if param_name in trial.params:
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(
trial.params[param_name])
else:
param_value = None
values[param_name].append(param_value)
if constraints_enabled:
assert violations is not None
constraint = trial.system_attrs.get(_CONSTRAINTS_KEY)
if constraint is None:
warnings.warn(
f"Trial {trial.number} does not have constraint values."
" It will be treated as a lower priority than other trials."
)
violation = float("inf")
else:
# Violation values of infeasible dimensions are summed up.
violation = sum(v for v in constraint if v > 0)
violations.append(violation)
return values, scores, violations
def _get_parallel_coordinate_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(title="Parallel Coordinate Plot")
reverse_scale = _is_reverse_scale(study, target)
trials = _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return go.Figure(data=[], layout=layout)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is not None:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError("Parameter {} does not exist in your study.".format(input_p_name))
all_params = set(params)
sorted_params = sorted(all_params)
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
skipped_trial_ids = _get_skipped_trial_numbers(trials, sorted_params)
objectives = tuple([target(t) for t in trials if t.number not in skipped_trial_ids])
if len(objectives) == 0:
_logger.warning("Your study has only completed trials with missing parameters.")
return go.Figure(data=[], layout=layout)
dims: List[Dict[str, Any]] = [
{
"label": target_name,
"values": objectives,
"range": (min(objectives), max(objectives)),
}
]
numeric_cat_params_indices: List[int] = []
for dim_index, p_name in enumerate(sorted_params, start=1):
values = []
for t in trials:
if t.number in skipped_trial_ids:
continue
if p_name in t.params:
values.append(t.params[p_name])
if _is_log_scale(trials, p_name):
values = [math.log10(v) for v in values]
min_value = min(values)
max_value = max(values)
tickvals = list(range(math.ceil(min_value), math.ceil(max_value)))
if min_value not in tickvals:
tickvals = [min_value] + tickvals
if max_value not in tickvals:
tickvals = tickvals + [max_value]
dim = {
"label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]),
"values": tuple(values),
"range": (min_value, max_value),
"tickvals": tickvals,
"ticktext": ["{:.3g}".format(math.pow(10, x)) for x in tickvals],
}
elif _is_categorical(trials, p_name):
vocab: DefaultDict[str, int] = defaultdict(lambda: len(vocab))
if _is_numerical(trials, p_name):
_ = [vocab[v] for v in sorted(values)]
values = [vocab[v] for v in values]
ticktext = list(sorted(vocab.keys()))
numeric_cat_params_indices.append(dim_index)
else:
values = [vocab[v] for v in values]
ticktext = list(sorted(vocab.keys(), key=lambda x: vocab[x]))
dim = {
"label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]),
"values": tuple(values),
"range": (min(values), max(values)),
"tickvals": list(range(len(vocab))),
"ticktext": ticktext,
}
else:
dim = {
"label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]),
"values": tuple(values),
"range": (min(values), max(values)),
}
dims.append(dim)
if numeric_cat_params_indices:
# np.lexsort consumes the sort keys the order from back to front.
# So the values of parameters have to be reversed the order.
idx = np.lexsort([dims[index]["values"] for index in numeric_cat_params_indices][::-1])
for dim in dims:
# Since the values are mapped to other categories by the index,
# the index will be swapped according to the sorted index of numeric params.
dim.update({"values": tuple(np.array(dim["values"])[idx])})
traces = [
go.Parcoords(
dimensions=dims,
labelangle=30,
labelside="bottom",
line={
"color": dims[0]["values"],
"colorscale": COLOR_SCALE,
"colorbar": {"title": target_name},
"showscale": True,
"reversescale": reverse_scale,
},
)
]
figure = go.Figure(data=traces, layout=layout)
return figure
def _get_observation_pairs(
study: Study,
param_names: List[str],
multivariate: bool,
constant_liar: bool = False, # TODO(hvy): Remove default value and fix unit tests.
) -> Tuple[Dict[str, List[Optional[float]]], List[Tuple[float, List[float]]]]:
"""Get observation pairs from the study.
This function collects observation pairs from the complete or pruned trials of the study.
In addition, if ``constant_liar`` is :obj:`True`, the running trials are considered.
The values for trials that don't contain the parameter in the ``param_names`` are skipped.
An observation pair fundamentally consists of a parameter value and an objective value.
However, due to the pruning mechanism of Optuna, final objective values are not always
available. Therefore, this function uses intermediate values in addition to the final
ones, and reports the value with its step count as ``(-step, value)``.
Consequently, the structure of the observation pair is as follows:
``(param_value, (-step, value))``.
The second element of an observation pair is used to rank observations in
``_split_observation_pairs`` method (i.e., observations are sorted lexicographically by
``(-step, value)``).
"""
if len(param_names) > 1:
assert multivariate
signs = []
for d in study.directions:
if d == StudyDirection.MINIMIZE:
signs.append(1)
else:
signs.append(-1)
states: Tuple[TrialState, ...]
if constant_liar:
states = (TrialState.COMPLETE, TrialState.PRUNED, TrialState.RUNNING)
else:
states = (TrialState.COMPLETE, TrialState.PRUNED)
scores = []
values: Dict[str, List[Optional[float]]] = {param_name: [] for param_name in param_names}
for trial in study.get_trials(deepcopy=False, states=states):
# If ``multivariate`` = True and ``group`` = True, we ignore the trials that are not
# included in each subspace.
# If ``multivariate`` = False, we skip the check.
if multivariate and any([param_name not in trial.params for param_name in param_names]):
continue
# We extract score from the trial.
if trial.state is TrialState.COMPLETE:
if trial.values is None:
continue
score = (-float("inf"), [sign * v for sign, v in zip(signs, trial.values)])
elif trial.state is TrialState.PRUNED:
if study._is_multi_objective():
continue
if len(trial.intermediate_values) > 0:
step, intermediate_value = max(trial.intermediate_values.items())
if math.isnan(intermediate_value):
score = (-step, [float("inf")])
else:
score = (-step, [signs[0] * intermediate_value])
else:
score = (float("inf"), [0.0])
elif trial.state is TrialState.RUNNING:
if study._is_multi_objective():
continue
assert constant_liar
score = (-float("inf"), [signs[0] * float("inf")])
else:
assert False
scores.append(score)
# We extract param_value from the trial.
for param_name in param_names:
raw_param_value = trial.params.get(param_name, None)
param_value: Optional[float]
if raw_param_value is not None:
distribution = trial.distributions[param_name]
param_value = distribution.to_internal_repr(trial.params[param_name])
else:
param_value = None
values[param_name].append(param_value)
return values, scores
def plot_param_importances(
study: Study,
evaluator: Optional[BaseImportanceEvaluator] = None,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
"""Plot hyperparameter importances.
Example:
The following code snippet shows how to plot hyperparameter importances.
.. plotly::
import optuna
def objective(trial):
x = trial.suggest_int("x", 0, 2)
y = trial.suggest_float("y", -1.0, 1.0)
z = trial.suggest_float("z", 0.0, 1.5)
return x ** 2 + y ** 3 - z ** 4
sampler = optuna.samplers.RandomSampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=100)
fig = optuna.visualization.plot_param_importances(study)
fig.show()
.. seealso::
This function visualizes the results of :func:`optuna.importance.get_param_importances`.
Args:
study:
An optimized study.
evaluator:
An importance evaluator object that specifies which algorithm to base the importance
assessment on.
Defaults to
:class:`~optuna.importance.FanovaImportanceEvaluator`.
params:
A list of names of parameters to assess.
If :obj:`None`, all parameters that are present in all of the completed trials are
assessed.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective
optimization. For example, to get the hyperparameter importance of the first
objective, use ``target=lambda t: t.values[0]`` for the target parameter.
target_name:
Target's name to display on the axis label.
Returns:
A :class:`plotly.graph_objs.Figure` object.
"""
_imports.check()
_check_plot_args(study, target, target_name)
layout = go.Layout(
title="Hyperparameter Importances",
xaxis={"title": f"Importance for {target_name}"},
yaxis={"title": "Hyperparameter"},
showlegend=False,
)
# Importances cannot be evaluated without completed trials.
# Return an empty figure for consistency with other visualization functions.
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
logger.warning("Study instance does not contain completed trials.")
return go.Figure(data=[], layout=layout)
importances = optuna.importance.get_param_importances(study,
evaluator=evaluator,
params=params,
target=target)
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
importance_labels = [
f"{val:.2f}" if val >= 0.01 else "<0.01" for val in importance_values
]
fig = go.Figure(
data=[
go.Bar(
x=importance_values,
y=param_names,
text=importance_labels,
textposition="outside",
cliponaxis=False, # Ensure text is not clipped.
hovertemplate=[
_make_hovertext(param_name, importance, study)
for param_name, importance in importances.items()
],
marker_color=plotly.colors.sequential.Blues[-4],
orientation="h",
)
],
layout=layout,
)
return fig
def _get_pareto_front_info(
study: Study,
target_names: Optional[List[str]] = None,
include_dominated_trials: bool = True,
axis_order: Optional[List[int]] = None,
constraints_func: Optional[Callable[[FrozenTrial],
Sequence[float]]] = None,
targets: Optional[Callable[[FrozenTrial], Sequence[float]]] = None,
) -> _ParetoFrontInfo:
if axis_order is not None:
warnings.warn(
"`axis_order` has been deprecated in v3.0.0. "
"This feature will be removed in v5.0.0. "
"See https://github.com/optuna/optuna/releases/tag/v3.0.0.",
DeprecationWarning,
)
if targets is not None and axis_order is not None:
raise ValueError(
"Using both `targets` and `axis_order` is not supported. "
"Use either `targets` or `axis_order`.")
if constraints_func is not None:
feasible_trials = []
infeasible_trials = []
for trial in study.get_trials(states=(TrialState.COMPLETE, )):
if all(map(lambda x: x <= 0.0, constraints_func(trial))):
feasible_trials.append(trial)
else:
infeasible_trials.append(trial)
best_trials = _get_pareto_front_trials_by_trials(
feasible_trials, study.directions)
if include_dominated_trials:
non_best_trials = _get_non_pareto_front_trials(
feasible_trials, best_trials)
else:
non_best_trials = []
if len(best_trials) == 0:
_logger.warning(
"Your study does not have any completed and feasible trials.")
else:
best_trials = study.best_trials
if len(best_trials) == 0:
_logger.warning("Your study does not have any completed trials.")
if include_dominated_trials:
non_best_trials = _get_non_pareto_front_trials(
study.get_trials(deepcopy=False), best_trials)
else:
non_best_trials = []
infeasible_trials = []
_targets = targets
if _targets is None:
if len(study.directions) in (2, 3):
_targets = _targets_default
else:
raise ValueError(
"`plot_pareto_front` function only supports 2 or 3 objective"
" studies when using `targets` is `None`. Please use `targets`"
" if your objective studies have more than 3 objectives.")
def _make_trials_with_values(
trials: Optional[List[FrozenTrial]],
targets: Callable[[FrozenTrial], Sequence[float]],
) -> Optional[Sequence[Tuple[FrozenTrial, Sequence[float]]]]:
if trials is None:
return None
return [(trial, targets(trial)) for trial in trials]
best_trials_with_values = _make_trials_with_values(best_trials, _targets)
non_best_trials_with_values = _make_trials_with_values(
non_best_trials, _targets)
infeasible_trials_with_values = _make_trials_with_values(
infeasible_trials, _targets)
def _infer_n_targets(
trials_with_values: Optional[Sequence[Tuple[FrozenTrial,
Sequence[float]]]]
) -> Optional[int]:
if trials_with_values is not None and len(trials_with_values) > 0:
if not isinstance(trials_with_values[0][1],
collections.abc.Sequence):
raise ValueError(
"`targets` should return a sequence of target values."
" your `targets` returns {}".format(
type(trials_with_values[0][1])))
return len(trials_with_values[0][1])
return None
# Check for `non_best_trials_with_values` can be skipped, because if `best_trials_with_values`
# is empty, then `non_best_trials_with_values` will also be empty.
n_targets = _infer_n_targets(best_trials_with_values) or _infer_n_targets(
infeasible_trials_with_values)
if n_targets is None:
if target_names is not None:
n_targets = len(target_names)
elif targets is None:
n_targets = len(study.directions)
else:
raise ValueError(
"If `targets` is specified for empty studies, `target_names` must be specified."
)
if target_names is None:
target_names = [f"Objective {i}" for i in range(n_targets)]
elif len(target_names) != n_targets:
raise ValueError(
f"The length of `target_names` is supposed to be {n_targets}.")
if axis_order is None:
axis_order = list(range(n_targets))
else:
if len(axis_order) != n_targets:
raise ValueError(
f"Size of `axis_order` {axis_order}. Expect: {n_targets}, "
f"Actual: {len(axis_order)}.")
if len(set(axis_order)) != n_targets:
raise ValueError(
f"Elements of given `axis_order` {axis_order} are not unique!."
)
if max(axis_order) > n_targets - 1:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} "
f"higher than {n_targets - 1}.")
if min(axis_order) < 0:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} "
"lower than 0.")
return _ParetoFrontInfo(
n_targets=n_targets,
target_names=target_names,
best_trials_with_values=best_trials_with_values,
non_best_trials_with_values=non_best_trials_with_values,
infeasible_trials_with_values=infeasible_trials_with_values,
axis_order=axis_order,
)
