def test_filter_inf_trials_message(caplog: LogCaptureFixture,
with_message: bool) -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={"x": 1.0},
distributions={"x": FloatDistribution(0.0, 1.0)},
))
study.add_trial(
create_trial(
value=float("inf"),
params={"x": 0.0},
distributions={"x": FloatDistribution(0.0, 1.0)},
))
optuna.logging.enable_propagation()
_filter_nonfinite(study.get_trials(states=(TrialState.COMPLETE, )),
with_message=with_message)
msg = "Trial 1 is omitted in visualization because its objective value is inf or nan."
if with_message:
assert msg in caplog.text
n_filtered_as_inf = 0
for record in caplog.records:
if record.msg == msg:
assert record.levelno == logging.WARNING
n_filtered_as_inf += 1
assert n_filtered_as_inf == 1
else:
assert msg not in caplog.text
def test_filter_inf_trials_multiobjective(value: float,
objective_selected: int,
expected: int) -> None:
study = create_study(directions=["minimize", "maximize"])
study.add_trial(
create_trial(
values=[0.0, 1.0],
params={"x": 1.0},
distributions={"x": FloatDistribution(0.0, 1.0)},
))
study.add_trial(
create_trial(
values=[0.0, value],
params={"x": 0.0},
distributions={"x": FloatDistribution(0.0, 1.0)},
))
study.add_trial(
create_trial(
values=[value, value],
params={"x": 0.0},
distributions={"x": FloatDistribution(0.0, 1.0)},
))
def _target(t: FrozenTrial) -> float:
return t.values[objective_selected]
trials = _filter_nonfinite(
study.get_trials(states=(TrialState.COMPLETE, )), target=_target)
assert len(trials) == expected
assert all([t.number == num for t, num in zip(trials, range(expected))])
def _get_edf_plot(
studies: List[Study],
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
# Set up the graph style.
plt.style.use(
"ggplot") # Use ggplot style sheet for similar outputs to plotly.
_, ax = plt.subplots()
ax.set_title("Empirical Distribution Function Plot")
ax.set_xlabel(target_name)
ax.set_ylabel("Cumulative Probability")
ax.set_ylim(0, 1)
cmap = plt.get_cmap("tab20") # Use tab20 colormap for multiple line plots.
# Prepare data for plotting.
if len(studies) == 0:
_logger.warning("There are no studies.")
return ax
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
all_values: List[np.ndarray] = []
for study in studies:
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
values = np.array([target(trial) for trial in trials])
all_values.append(values)
if all(len(values) == 0 for values in all_values):
_logger.warning("There are no complete trials.")
return ax
min_x_value = np.min(np.concatenate(all_values))
max_x_value = np.max(np.concatenate(all_values))
x_values = np.linspace(min_x_value, max_x_value, 100)
# Draw multiple line plots.
for i, (values, study) in enumerate(zip(all_values, studies)):
y_values = np.sum(values[:, np.newaxis] <= x_values,
axis=0) / values.size
ax.plot(x_values,
y_values,
color=cmap(i),
alpha=0.7,
label=study.study_name)
if len(studies) >= 2:
ax.legend()
return ax
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
trials = []
for trial in _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions,
transform_log=False,
transform_step=False)
n_trials = len(trials)
self._trans_params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
self._trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
self._trans_params[trial_idx] = trans.transform(trial.params)
self._trans_values[
trial_idx] = trial.value if target is None else target(trial)
encoded_column_to_column = trans.encoded_column_to_column
if self._trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
forest = self._forest
forest.fit(self._trans_params, self._trans_values)
feature_importances = forest.feature_importances_
feature_importances_reduced = numpy.zeros(len(distributions))
numpy.add.at(feature_importances_reduced, encoded_column_to_column,
feature_importances)
param_importances = OrderedDict()
self._param_names = list(distributions.keys())
for i in feature_importances_reduced.argsort()[::-1]:
param_importances[
self._param_names[i]] = feature_importances_reduced[i].item()
return param_importances
def _get_contour_info(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> _ContourInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
sorted_params = []
elif params is None:
sorted_params = sorted(all_params)
else:
if len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
"Parameter {} does not exist in your study.".format(
input_p_name))
sorted_params = sorted(set(params))
sub_plot_infos: List[List[_SubContourInfo]]
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plot_info = _get_contour_subplot_info(trials, x_param, y_param,
target)
sub_plot_infos = [[sub_plot_info]]
else:
sub_plot_infos = []
for i, y_param in enumerate(sorted_params):
sub_plot_infos.append([])
for x_param in sorted_params:
sub_plot_info = _get_contour_subplot_info(
trials, x_param, y_param, target)
sub_plot_infos[i].append(sub_plot_info)
reverse_scale = _is_reverse_scale(study, target)
return _ContourInfo(
sorted_params=sorted_params,
sub_plot_infos=sub_plot_infos,
reverse_scale=reverse_scale,
target_name=target_name,
)
def _get_edf_info(
study: Union[Study, Sequence[Study]],
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> _EDFInfo:
if isinstance(study, Study):
studies = [study]
else:
studies = list(study)
_check_plot_args(studies, target, target_name)
if len(studies) == 0:
_logger.warning("There are no studies.")
return _EDFInfo(lines=[], x_values=np.array([]))
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
study_names = []
all_values: List[np.ndarray] = []
for study in studies:
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
values = np.array([target(trial) for trial in trials])
all_values.append(values)
study_names.append(study.study_name)
if all(len(values) == 0 for values in all_values):
_logger.warning("There are no complete trials.")
return _EDFInfo(lines=[], x_values=np.array([]))
min_x_value = np.min(np.concatenate(all_values))
max_x_value = np.max(np.concatenate(all_values))
x_values = np.linspace(min_x_value, max_x_value, NUM_SAMPLES_X_AXIS)
edf_line_info_list = []
for (study_name, values) in zip(study_names, all_values):
y_values = np.sum(values[:, np.newaxis] <= x_values,
axis=0) / values.size
edf_line_info_list.append(
_EDFLineInfo(study_name=study_name, y_values=y_values))
return _EDFInfo(lines=edf_line_info_list, x_values=x_values)
def _get_edf_plot(
studies: List[Study],
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(
title="Empirical Distribution Function Plot",
xaxis={"title": target_name},
yaxis={"title": "Cumulative Probability"},
)
if len(studies) == 0:
_logger.warning("There are no studies.")
return go.Figure(data=[], layout=layout)
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
all_values: List[np.ndarray] = []
for study in studies:
trials = _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
)
values = np.array([target(trial) for trial in trials])
all_values.append(values)
if all(len(values) == 0 for values in all_values):
_logger.warning("There are no complete trials.")
return go.Figure(data=[], layout=layout)
min_x_value = np.min(np.concatenate(all_values))
max_x_value = np.max(np.concatenate(all_values))
x_values = np.linspace(min_x_value, max_x_value, 100)
traces = []
for values, study in zip(all_values, studies):
y_values = np.sum(values[:, np.newaxis] <= x_values, axis=0) / values.size
traces.append(go.Scatter(x=x_values, y=y_values, name=study.study_name, mode="lines"))
figure = go.Figure(data=traces, layout=layout)
figure.update_yaxes(range=[0, 1])
return figure
def test_filter_inf_trials(value: float, expected: int) -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={"x": 1.0},
distributions={"x": FloatDistribution(0.0, 1.0)},
))
study.add_trial(
create_trial(
value=value,
params={"x": 0.0},
distributions={"x": FloatDistribution(0.0, 1.0)},
))
trials = _filter_nonfinite(
study.get_trials(states=(TrialState.COMPLETE, )))
assert len(trials) == expected
assert all([t.number == num for t, num in zip(trials, range(expected))])
def _get_importances_info(
study: Study,
evaluator: Optional[BaseImportanceEvaluator],
params: Optional[List[str]],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> _ImportancesInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
logger.warning("Study instance does not contain completed trials.")
return _ImportancesInfo(
importance_values=[],
param_names=[],
importance_labels=[],
target_name=target_name,
)
importances = optuna.importance.get_param_importances(study,
evaluator=evaluator,
params=params,
target=target)
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
importance_labels = [
f"{val:.2f}" if val >= 0.01 else "<0.01" for val in importance_values
]
return _ImportancesInfo(
importance_values=importance_values,
param_names=param_names,
importance_labels=importance_labels,
target_name=target_name,
)
def _get_slice_plot_info(
study: Study,
params: Optional[List[str]],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> _SlicePlotInfo:
_check_plot_args(study, target, target_name)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return _SlicePlotInfo(target_name, [])
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
f"Parameter {input_p_name} does not exist in your study.")
sorted_params = sorted(set(params))
return _SlicePlotInfo(
target_name=target_name,
subplots=[
_get_slice_subplot_info(
trials=trials,
param=param,
target=target,
log_scale=_is_log_scale(trials, param),
numerical=_is_numerical(trials, param),
) for param in sorted_params
],
)
def evaluate(
self,
study: Study,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
) -> Dict[str, float]:
if target is None and study._is_multi_objective():
raise ValueError(
"If the `study` is being used for multi-objective optimization, "
"please specify the `target`. For example, use "
"`target=lambda t: t.values[0]` for the first objective value."
)
distributions = _get_distributions(study, params)
if len(distributions) == 0:
return OrderedDict()
# fANOVA does not support parameter distributions with a single value.
# However, there is no reason to calculate parameter importance in such case anyway,
# since it will always be 0 as the parameter is constant in the objective function.
zero_importances = {
name: 0.0
for name, dist in distributions.items() if dist.single()
}
distributions = {
name: dist
for name, dist in distributions.items() if not dist.single()
}
trials = []
for trial in _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target):
if any(name not in trial.params for name in distributions.keys()):
continue
trials.append(trial)
trans = _SearchSpaceTransform(distributions,
transform_log=False,
transform_step=False)
n_trials = len(trials)
trans_params = numpy.empty((n_trials, trans.bounds.shape[0]),
dtype=numpy.float64)
trans_values = numpy.empty(n_trials, dtype=numpy.float64)
for trial_idx, trial in enumerate(trials):
trans_params[trial_idx] = trans.transform(trial.params)
trans_values[
trial_idx] = trial.value if target is None else target(trial)
trans_bounds = trans.bounds
column_to_encoded_columns = trans.column_to_encoded_columns
if trans_params.size == 0: # `params` were given but as an empty list.
return OrderedDict()
# Many (deep) copies of the search spaces are required during the tree traversal and using
# Optuna distributions will create a bottleneck.
# Therefore, search spaces (parameter distributions) are represented by a single
# `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical
# or not.
evaluator = self._evaluator
evaluator.fit(
X=trans_params,
y=trans_values,
search_spaces=trans_bounds,
column_to_encoded_columns=column_to_encoded_columns,
)
importances = {}
for i, name in enumerate(distributions.keys()):
importance, _ = evaluator.get_importance((i, ))
importances[name] = importance
importances = {**importances, **zero_importances}
total_importance = sum(importances.values())
for name in importances:
importances[name] /= total_importance
sorted_importances = OrderedDict(
reversed(
sorted(
importances.items(),
key=lambda name_and_importance: name_and_importance[1])))
return sorted_importances
def _get_parallel_coordinate_info(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> _ParallelCoordinateInfo:
reverse_scale = _is_reverse_scale(study, target)
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is not None:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
"Parameter {} does not exist in your study.".format(
input_p_name))
all_params = set(params)
sorted_params = sorted(all_params)
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
skipped_trial_numbers = _get_skipped_trial_numbers(trials, sorted_params)
objectives = tuple(
[target(t) for t in trials if t.number not in skipped_trial_numbers])
# The value of (0, 0) is a dummy range. It is ignored when we plot.
objective_range = (min(objectives),
max(objectives)) if len(objectives) > 0 else (0, 0)
dim_objective = _DimensionInfo(
label=target_name,
values=objectives,
range=objective_range,
is_log=False,
is_cat=False,
tickvals=[],
ticktext=[],
)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return _ParallelCoordinateInfo(
dim_objective=dim_objective,
dims_params=[],
reverse_scale=reverse_scale,
target_name=target_name,
)
if len(objectives) == 0:
_logger.warning(
"Your study has only completed trials with missing parameters.")
return _ParallelCoordinateInfo(
dim_objective=dim_objective,
dims_params=[],
reverse_scale=reverse_scale,
target_name=target_name,
)
numeric_cat_params_indices: List[int] = []
dims = []
for dim_index, p_name in enumerate(sorted_params, start=1):
values = []
for t in trials:
if t.number in skipped_trial_numbers:
continue
if p_name in t.params:
values.append(t.params[p_name])
if _is_log_scale(trials, p_name):
values = [math.log10(v) for v in values]
min_value = min(values)
max_value = max(values)
tickvals = list(range(math.ceil(min_value), math.ceil(max_value)))
if min_value not in tickvals:
tickvals = [min_value] + tickvals
if max_value not in tickvals:
tickvals = tickvals + [max_value]
dim = _DimensionInfo(
label=_truncate_label(p_name),
values=tuple(values),
range=(min_value, max_value),
is_log=True,
is_cat=False,
tickvals=tickvals,
ticktext=["{:.3g}".format(math.pow(10, x)) for x in tickvals],
)
elif _is_categorical(trials, p_name):
vocab: DefaultDict[str, int] = defaultdict(lambda: len(vocab))
if _is_numerical(trials, p_name):
_ = [vocab[v] for v in sorted(values)]
values = [vocab[v] for v in values]
ticktext = list(sorted(vocab.keys()))
numeric_cat_params_indices.append(dim_index)
else:
values = [vocab[v] for v in values]
ticktext = list(sorted(vocab.keys(), key=lambda x: vocab[x]))
dim = _DimensionInfo(
label=_truncate_label(p_name),
values=tuple(values),
range=(min(values), max(values)),
is_log=False,
is_cat=True,
tickvals=list(range(len(vocab))),
ticktext=ticktext,
)
else:
dim = _DimensionInfo(
label=_truncate_label(p_name),
values=tuple(values),
range=(min(values), max(values)),
is_log=False,
is_cat=False,
tickvals=[],
ticktext=[],
)
dims.append(dim)
if numeric_cat_params_indices:
dims.insert(0, dim_objective)
# np.lexsort consumes the sort keys the order from back to front.
# So the values of parameters have to be reversed the order.
idx = np.lexsort(
[dims[index].values for index in numeric_cat_params_indices][::-1])
updated_dims = []
for dim in dims:
# Since the values are mapped to other categories by the index,
# the index will be swapped according to the sorted index of numeric params.
updated_dims.append(
_DimensionInfo(
label=dim.label,
values=tuple(np.array(dim.values)[idx]),
range=dim.range,
is_log=dim.is_log,
is_cat=dim.is_cat,
tickvals=dim.tickvals,
ticktext=dim.ticktext,
))
dim_objective = updated_dims[0]
dims = updated_dims[1:]
return _ParallelCoordinateInfo(
dim_objective=dim_objective,
dims_params=dims,
reverse_scale=reverse_scale,
target_name=target_name,
)
def _get_parallel_coordinate_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(title="Parallel Coordinate Plot")
reverse_scale = _is_reverse_scale(study, target)
trials = _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return go.Figure(data=[], layout=layout)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is not None:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError("Parameter {} does not exist in your study.".format(input_p_name))
all_params = set(params)
sorted_params = sorted(all_params)
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
skipped_trial_ids = _get_skipped_trial_numbers(trials, sorted_params)
objectives = tuple([target(t) for t in trials if t.number not in skipped_trial_ids])
if len(objectives) == 0:
_logger.warning("Your study has only completed trials with missing parameters.")
return go.Figure(data=[], layout=layout)
dims: List[Dict[str, Any]] = [
{
"label": target_name,
"values": objectives,
"range": (min(objectives), max(objectives)),
}
]
numeric_cat_params_indices: List[int] = []
for dim_index, p_name in enumerate(sorted_params, start=1):
values = []
for t in trials:
if t.number in skipped_trial_ids:
continue
if p_name in t.params:
values.append(t.params[p_name])
if _is_log_scale(trials, p_name):
values = [math.log10(v) for v in values]
min_value = min(values)
max_value = max(values)
tickvals = list(range(math.ceil(min_value), math.ceil(max_value)))
if min_value not in tickvals:
tickvals = [min_value] + tickvals
if max_value not in tickvals:
tickvals = tickvals + [max_value]
dim = {
"label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]),
"values": tuple(values),
"range": (min_value, max_value),
"tickvals": tickvals,
"ticktext": ["{:.3g}".format(math.pow(10, x)) for x in tickvals],
}
elif _is_categorical(trials, p_name):
vocab: DefaultDict[str, int] = defaultdict(lambda: len(vocab))
if _is_numerical(trials, p_name):
_ = [vocab[v] for v in sorted(values)]
values = [vocab[v] for v in values]
ticktext = list(sorted(vocab.keys()))
numeric_cat_params_indices.append(dim_index)
else:
values = [vocab[v] for v in values]
ticktext = list(sorted(vocab.keys(), key=lambda x: vocab[x]))
dim = {
"label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]),
"values": tuple(values),
"range": (min(values), max(values)),
"tickvals": list(range(len(vocab))),
"ticktext": ticktext,
}
else:
dim = {
"label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]),
"values": tuple(values),
"range": (min(values), max(values)),
}
dims.append(dim)
if numeric_cat_params_indices:
# np.lexsort consumes the sort keys the order from back to front.
# So the values of parameters have to be reversed the order.
idx = np.lexsort([dims[index]["values"] for index in numeric_cat_params_indices][::-1])
for dim in dims:
# Since the values are mapped to other categories by the index,
# the index will be swapped according to the sorted index of numeric params.
dim.update({"values": tuple(np.array(dim["values"])[idx])})
traces = [
go.Parcoords(
dimensions=dims,
labelangle=30,
labelside="bottom",
line={
"color": dims[0]["values"],
"colorscale": COLOR_SCALE,
"colorbar": {"title": target_name},
"showscale": True,
"reversescale": reverse_scale,
},
)
]
figure = go.Figure(data=traces, layout=layout)
return figure
def plot_param_importances(
study: Study,
evaluator: Optional[BaseImportanceEvaluator] = None,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
"""Plot hyperparameter importances.
Example:
The following code snippet shows how to plot hyperparameter importances.
.. plotly::
import optuna
def objective(trial):
x = trial.suggest_int("x", 0, 2)
y = trial.suggest_float("y", -1.0, 1.0)
z = trial.suggest_float("z", 0.0, 1.5)
return x ** 2 + y ** 3 - z ** 4
sampler = optuna.samplers.RandomSampler(seed=10)
study = optuna.create_study(sampler=sampler)
study.optimize(objective, n_trials=100)
fig = optuna.visualization.plot_param_importances(study)
fig.show()
.. seealso::
This function visualizes the results of :func:`optuna.importance.get_param_importances`.
Args:
study:
An optimized study.
evaluator:
An importance evaluator object that specifies which algorithm to base the importance
assessment on.
Defaults to
:class:`~optuna.importance.FanovaImportanceEvaluator`.
params:
A list of names of parameters to assess.
If :obj:`None`, all parameters that are present in all of the completed trials are
assessed.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective
optimization. For example, to get the hyperparameter importance of the first
objective, use ``target=lambda t: t.values[0]`` for the target parameter.
target_name:
Target's name to display on the axis label.
Returns:
A :class:`plotly.graph_objs.Figure` object.
"""
_imports.check()
_check_plot_args(study, target, target_name)
layout = go.Layout(
title="Hyperparameter Importances",
xaxis={"title": f"Importance for {target_name}"},
yaxis={"title": "Hyperparameter"},
showlegend=False,
)
# Importances cannot be evaluated without completed trials.
# Return an empty figure for consistency with other visualization functions.
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
logger.warning("Study instance does not contain completed trials.")
return go.Figure(data=[], layout=layout)
importances = optuna.importance.get_param_importances(study,
evaluator=evaluator,
params=params,
target=target)
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
importance_labels = [
f"{val:.2f}" if val >= 0.01 else "<0.01" for val in importance_values
]
fig = go.Figure(
data=[
go.Bar(
x=importance_values,
y=param_names,
text=importance_labels,
textposition="outside",
cliponaxis=False, # Ensure text is not clipped.
hovertemplate=[
_make_hovertext(param_name, importance, study)
for param_name, importance in importances.items()
],
marker_color=plotly.colors.sequential.Blues[-4],
orientation="h",
)
],
layout=layout,
)
return fig
def _get_param_importance_plot(
study: Study,
evaluator: Optional[BaseImportanceEvaluator] = None,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
# Set up the graph style.
plt.style.use(
"ggplot") # Use ggplot style sheet for similar outputs to plotly.
fig, ax = plt.subplots()
ax.set_title("Hyperparameter Importances")
ax.set_xlabel(f"Importance for {target_name}")
ax.set_ylabel("Hyperparameter")
# Prepare data for plotting.
# Importances cannot be evaluated without completed trials.
# Return an empty figure for consistency with other visualization functions.
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
_logger.warning("Study instance does not contain completed trials.")
return ax
importances = optuna.importance.get_param_importances(study,
evaluator=evaluator,
params=params,
target=target)
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
pos = np.arange(len(param_names))
# Draw horizontal bars.
ax.barh(
pos,
importance_values,
align="center",
color=cm.get_cmap("tab20c")(0),
tick_label=param_names,
)
renderer = fig.canvas.get_renderer()
for idx, val in enumerate(importance_values):
label = f" {val:.2f}" if val >= 0.01 else " <0.01"
text = ax.text(val, idx, label, va="center")
# Sometimes horizontal axis needs to be re-scaled
# to avoid text going over plot area.
bbox = text.get_window_extent(renderer)
bbox = bbox.transformed(ax.transData.inverted())
_, plot_xmax = ax.get_xlim()
bbox_xmax = bbox.xmax
if bbox_xmax > plot_xmax:
ax.set_xlim(xmax=AXES_PADDING_RATIO * bbox_xmax)
return ax
def test_filter_nonfinite_with_invalid_target() -> None:
study = prepare_study_with_trials()
trials = study.get_trials(states=(TrialState.COMPLETE,))
with pytest.raises(ValueError):
_filter_nonfinite(trials, target=lambda t: "invalid target") # type: ignore
def _get_contour_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
# Calculate basic numbers for plotting.
trials = _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
_, ax = plt.subplots()
return ax
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
elif len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
_, ax = plt.subplots()
return ax
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError("Parameter {} does not exist in your study.".format(input_p_name))
sorted_params = sorted(set(params))
n_params = len(sorted_params)
plt.style.use("ggplot") # Use ggplot style sheet for similar outputs to plotly.
if n_params == 2:
# Set up the graph style.
fig, axs = plt.subplots()
axs.set_title("Contour Plot")
cmap = _set_cmap(study, target)
contour_point_num = 100
# Prepare data and draw contour plots.
if params:
x_param = params[0]
y_param = params[1]
else:
x_param = sorted_params[0]
y_param = sorted_params[1]
cs = _generate_contour_subplot(
trials, x_param, y_param, axs, cmap, contour_point_num, target
)
if isinstance(cs, ContourSet):
axcb = fig.colorbar(cs)
axcb.set_label(target_name)
else:
# Set up the graph style.
fig, axs = plt.subplots(n_params, n_params)
fig.suptitle("Contour Plot")
cmap = _set_cmap(study, target)
contour_point_num = 100
# Prepare data and draw contour plots.
cs_list = []
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
ax = axs[y_i, x_i]
cs = _generate_contour_subplot(
trials, x_param, y_param, ax, cmap, contour_point_num, target
)
if isinstance(cs, ContourSet):
cs_list.append(cs)
if cs_list:
axcb = fig.colorbar(cs_list[0], ax=axs)
axcb.set_label(target_name)
return axs
def _get_contour_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(title="Contour Plot")
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return go.Figure(data=[], layout=layout)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
elif len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
return go.Figure(data=[], layout=layout)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
"Parameter {} does not exist in your study.".format(
input_p_name))
sorted_params = sorted(set(params))
padding_ratio = 0.05
param_values_range = {}
for p_name in sorted_params:
values = _get_param_values(trials, p_name)
min_value = min(values)
max_value = max(values)
if _is_log_scale(trials, p_name):
padding = (math.log10(max_value) -
math.log10(min_value)) * padding_ratio
min_value = math.pow(10, math.log10(min_value) - padding)
max_value = math.pow(10, math.log10(max_value) + padding)
elif _is_numerical(trials, p_name):
padding = (max_value - min_value) * padding_ratio
min_value = min_value - padding
max_value = max_value + padding
else:
# Plotly>=4.12.0 draws contours using the indices of categorical variables instead of
# raw values and the range should be updated based on the cardinality of categorical
# variables. See https://github.com/optuna/optuna/issues/1967.
if version.parse(plotly.__version__) >= version.parse("4.12.0"):
span = len(set(values)) - 1
padding = span * padding_ratio
min_value = -padding
max_value = span + padding
param_values_range[p_name] = (min_value, max_value)
reverse_scale = _is_reverse_scale(study, target)
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plots = _generate_contour_subplot(trials, x_param, y_param,
reverse_scale,
param_values_range, target,
target_name)
figure = go.Figure(data=sub_plots, layout=layout)
figure.update_xaxes(title_text=x_param,
range=param_values_range[x_param])
figure.update_yaxes(title_text=y_param,
range=param_values_range[y_param])
if not _is_numerical(trials, x_param):
figure.update_xaxes(type="category")
if not _is_numerical(trials, y_param):
figure.update_yaxes(type="category")
if _is_log_scale(trials, x_param):
log_range = [math.log10(p) for p in param_values_range[x_param]]
figure.update_xaxes(range=log_range, type="log")
if _is_log_scale(trials, y_param):
log_range = [math.log10(p) for p in param_values_range[y_param]]
figure.update_yaxes(range=log_range, type="log")
else:
figure = make_subplots(rows=len(sorted_params),
cols=len(sorted_params),
shared_xaxes=True,
shared_yaxes=True)
figure.update_layout(layout)
showscale = True # showscale option only needs to be specified once
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
if x_param == y_param:
figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1)
else:
sub_plots = _generate_contour_subplot(
trials,
x_param,
y_param,
reverse_scale,
param_values_range,
target,
target_name,
)
contour = sub_plots[0]
scatter = sub_plots[1]
contour.update(
showscale=showscale) # showscale's default is True
if showscale:
showscale = False
figure.add_trace(contour, row=y_i + 1, col=x_i + 1)
figure.add_trace(scatter, row=y_i + 1, col=x_i + 1)
figure.update_xaxes(range=param_values_range[x_param],
row=y_i + 1,
col=x_i + 1)
figure.update_yaxes(range=param_values_range[y_param],
row=y_i + 1,
col=x_i + 1)
if not _is_numerical(trials, x_param):
figure.update_xaxes(type="category",
row=y_i + 1,
col=x_i + 1)
if not _is_numerical(trials, y_param):
figure.update_yaxes(type="category",
row=y_i + 1,
col=x_i + 1)
if _is_log_scale(trials, x_param):
log_range = [
math.log10(p) for p in param_values_range[x_param]
]
figure.update_xaxes(range=log_range,
type="log",
row=y_i + 1,
col=x_i + 1)
if _is_log_scale(trials, y_param):
log_range = [
math.log10(p) for p in param_values_range[y_param]
]
figure.update_yaxes(range=log_range,
type="log",
row=y_i + 1,
col=x_i + 1)
if x_i == 0:
figure.update_yaxes(title_text=y_param,
row=y_i + 1,
col=x_i + 1)
if y_i == len(sorted_params) - 1:
figure.update_xaxes(title_text=x_param,
row=y_i + 1,
col=x_i + 1)
return figure
def _get_parallel_coordinate_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "Axes":
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
reversescale = study.direction == StudyDirection.MINIMIZE
else:
reversescale = True
# Set up the graph style.
fig, ax = plt.subplots()
cmap = plt.get_cmap("Blues_r" if reversescale else "Blues")
ax.set_title("Parallel Coordinate Plot")
ax.spines["top"].set_visible(False)
ax.spines["bottom"].set_visible(False)
# Prepare data for plotting.
trials = _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return ax
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is not None:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError("Parameter {} does not exist in your study.".format(input_p_name))
all_params = set(params)
sorted_params = sorted(all_params)
skipped_trial_numbers = _get_skipped_trial_numbers(trials, sorted_params)
obj_org = [target(t) for t in trials if t.number not in skipped_trial_numbers]
if len(obj_org) == 0:
_logger.warning("Your study has only completed trials with missing parameters.")
return ax
obj_min = min(obj_org)
obj_max = max(obj_org)
obj_w = obj_max - obj_min
dims_obj_base = [[o] for o in obj_org]
cat_param_names = []
cat_param_values = []
cat_param_ticks = []
param_values = []
var_names = [target_name]
numeric_cat_params_indices: List[int] = []
for param_index, p_name in enumerate(sorted_params):
values = [t.params[p_name] for t in trials if t.number not in skipped_trial_numbers]
if _is_categorical(trials, p_name):
vocab = defaultdict(lambda: len(vocab)) # type: DefaultDict[str, int]
if _is_numerical(trials, p_name):
_ = [vocab[v] for v in sorted(values)]
numeric_cat_params_indices.append(param_index)
values = [vocab[v] for v in values]
cat_param_names.append(p_name)
vocab_item_sorted = sorted(vocab.items(), key=lambda x: x[1])
cat_param_values.append([v[0] for v in vocab_item_sorted])
cat_param_ticks.append([v[1] for v in vocab_item_sorted])
if _is_log_scale(trials, p_name):
values_for_lc = [np.log10(v) for v in values]
else:
values_for_lc = values
p_min = min(values_for_lc)
p_max = max(values_for_lc)
p_w = p_max - p_min
if p_w == 0.0:
center = obj_w / 2 + obj_min
for i in range(len(values)):
dims_obj_base[i].append(center)
else:
for i, v in enumerate(values_for_lc):
dims_obj_base[i].append((v - p_min) / p_w * obj_w + obj_min)
var_names.append(p_name if len(p_name) < 20 else "{}...".format(p_name[:17]))
param_values.append(values)
if numeric_cat_params_indices:
# np.lexsort consumes the sort keys the order from back to front.
# So the values of parameters have to be reversed the order.
sorted_idx = np.lexsort(
[param_values[index] for index in numeric_cat_params_indices][::-1]
)
# Since the values are mapped to other categories by the index,
# the index will be swapped according to the sorted index of numeric params.
param_values = [list(np.array(v)[sorted_idx]) for v in param_values]
# Draw multiple line plots and axes.
# Ref: https://stackoverflow.com/a/50029441
ax.set_xlim(0, len(sorted_params))
ax.set_ylim(obj_min, obj_max)
xs = [range(len(sorted_params) + 1) for _ in range(len(dims_obj_base))]
segments = [np.column_stack([x, y]) for x, y in zip(xs, dims_obj_base)]
lc = LineCollection(segments, cmap=cmap)
lc.set_array(np.asarray(obj_org))
axcb = fig.colorbar(lc, pad=0.1)
axcb.set_label(target_name)
plt.xticks(range(len(sorted_params) + 1), var_names, rotation=330)
for i, p_name in enumerate(sorted_params):
ax2 = ax.twinx()
ax2.set_ylim(min(param_values[i]), max(param_values[i]))
if _is_log_scale(trials, p_name):
ax2.set_yscale("log")
ax2.spines["top"].set_visible(False)
ax2.spines["bottom"].set_visible(False)
ax2.xaxis.set_visible(False)
ax2.spines["right"].set_position(("axes", (i + 1) / len(sorted_params)))
if p_name in cat_param_names:
idx = cat_param_names.index(p_name)
tick_pos = cat_param_ticks[idx]
tick_labels = cat_param_values[idx]
ax2.set_yticks(tick_pos)
ax2.set_yticklabels(tick_labels)
ax.add_collection(lc)
return ax
