def _get_optimization_history_plot(
study: Study,
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> "go.Figure":
layout = go.Layout(
title="Optimization History Plot",
xaxis={"title": "#Trials"},
yaxis={"title": target_name},
)
trials = [t for t in study.trials if t.state == TrialState.COMPLETE]
if len(trials) == 0:
_logger.warning("Study instance does not contain trials.")
return go.Figure(data=[], layout=layout)
if target is None:
if study.direction == StudyDirection.MINIMIZE:
best_values = [float("inf")]
else:
best_values = [-float("inf")]
comp = min if study.direction == StudyDirection.MINIMIZE else max
for trial in trials:
trial_value = trial.value
assert trial_value is not None # For mypy
best_values.append(comp(best_values[-1], trial_value))
best_values.pop(0)
traces = [
go.Scatter(
x=[t.number for t in trials],
y=[t.value for t in trials],
mode="markers",
name=target_name,
),
go.Scatter(x=[t.number for t in trials],
y=best_values,
name="Best Value"),
]
else:
traces = [
go.Scatter(
x=[t.number for t in trials],
y=[target(t) for t in trials],
mode="markers",
name=target_name,
),
]
figure = go.Figure(data=traces, layout=layout)
return figure
def _get_optimization_histories(
studies: List[Study],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
layout: "go.Layout",
) -> "go.Figure":
traces = []
for study in studies:
trials = study.get_trials(states=(TrialState.COMPLETE,))
if target is None:
if study.direction == StudyDirection.MINIMIZE:
best_values = np.minimum.accumulate([cast(float, t.value) for t in trials])
else:
best_values = np.maximum.accumulate([cast(float, t.value) for t in trials])
traces.append(
go.Scatter(
x=[t.number for t in trials],
y=[t.value for t in trials],
mode="markers",
name=target_name
if len(studies) == 1
else f"{target_name} of {study.study_name}",
)
)
traces.append(
go.Scatter(
x=[t.number for t in trials],
y=best_values,
name="Best Value"
if len(studies) == 1
else f"Best Value of {study.study_name}",
)
)
else:
traces.append(
go.Scatter(
x=[t.number for t in trials],
y=[target(t) for t in trials],
mode="markers",
name=target_name
if len(studies) == 1
else f"{target_name} of {study.study_name}",
)
)
figure = go.Figure(data=traces, layout=layout)
figure.update_layout(width=1000, height=400)
return figure
def _get_contour_subplot(
info: _SubContourInfo,
reverse_scale: bool,
target_name: str = "Objective Value",
) -> Tuple["Contour", "Scatter"]:
x_indices = info.xaxis.indices
y_indices = info.yaxis.indices
x_values = []
y_values = []
for x_value, y_value in zip(info.xaxis.values, info.yaxis.values):
if x_value is not None and y_value is not None:
x_values.append(x_value)
y_values.append(y_value)
z_values = [[float("nan") for _ in range(len(info.xaxis.indices))]
for _ in range(len(info.yaxis.indices))]
for (x_i, y_i), z_value in info.z_values.items():
z_values[y_i][x_i] = z_value
if len(x_indices) < 2 or len(y_indices) < 2:
return go.Contour(), go.Scatter()
contour = go.Contour(
x=x_indices,
y=y_indices,
z=z_values,
colorbar={"title": target_name},
colorscale=COLOR_SCALE,
connectgaps=True,
contours_coloring="heatmap",
hoverinfo="none",
line_smoothing=1.3,
reversescale=reverse_scale,
)
scatter = go.Scatter(
x=x_values,
y=y_values,
marker={
"line": {
"width": 2.0,
"color": "Grey"
},
"color": "black"
},
mode="markers",
showlegend=False,
)
return contour, scatter
def _get_intermediate_plot(study: Study) -> "go.Figure":
layout = go.Layout(
title="Intermediate Values Plot",
xaxis={"title": "Step"},
yaxis={"title": "Intermediate Value"},
showlegend=False,
)
info = _get_intermediate_plot_info(study)
trial_infos = info.trial_infos
if len(trial_infos) == 0:
return go.Figure(data=[], layout=layout)
traces = [
go.Scatter(
x=tuple((x for x, _ in tinfo.sorted_intermediate_values)),
y=tuple((y for _, y in tinfo.sorted_intermediate_values)),
mode="lines+markers",
marker={"maxdisplayed": 10},
name="Trial{}".format(tinfo.trial_number),
) for tinfo in trial_infos
]
return go.Figure(data=traces, layout=layout)
def _get_pareto_front_2d(
study: MultiObjectiveStudy,
names: Optional[List[str]],
include_dominated_trials: bool = False) -> "go.Figure":
if names is None:
names = ["Objective 0", "Objective 1"]
elif len(names) != 2:
raise ValueError("The length of `names` is supposed to be 2.")
trials = study.get_pareto_front_trials()
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
point_colors = ["blue"] * len(trials)
if include_dominated_trials:
non_pareto_trials = _get_non_pareto_front_trials(study, trials)
point_colors += ["red"] * len(non_pareto_trials)
trials += non_pareto_trials
data = go.Scatter(
x=[t.values[0] for t in trials],
y=[t.values[1] for t in trials],
text=[_make_hovertext(t) for t in trials],
mode="markers",
hovertemplate="%{text}<extra></extra>",
marker={"color": point_colors},
)
layout = go.Layout(title="Pareto-front Plot",
xaxis_title=names[0],
yaxis_title=names[1])
return go.Figure(data=data, layout=layout)
def _generate_slice_subplot(
trials: List[FrozenTrial],
param: str,
target: Optional[Callable[[FrozenTrial], float]],
) -> "Scatter":
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
return go.Scatter(
x=[t.params[param] for t in trials if param in t.params],
y=[target(t) for t in trials if param in t.params],
mode="markers",
marker={
"line": {
"width": 0.5,
"color": "Grey"
},
"color": [t.number for t in trials if param in t.params],
"colorscale": COLOR_SCALE,
"colorbar": {
"title": "Trial",
"x":
1.0, # Offset the colorbar position with a fixed width `xpad`.
"xpad": 40,
},
},
showlegend=False,
)
def _get_optimization_history_plot(
study: Study,
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
) -> "go.Figure":
layout = go.Layout(
title="Optimization History Plot",
xaxis={"title": "#Trials"},
yaxis={"title": target_name},
)
trials = [t for t in study.trials if t.state == TrialState.COMPLETE]
if len(trials) == 0:
_logger.warning("Study instance does not contain trials.")
return go.Figure(data=[], layout=layout)
if target is None:
if study.direction == StudyDirection.MINIMIZE:
best_values = np.minimum.accumulate([t.value for t in trials])
else:
best_values = np.maximum.accumulate([t.value for t in trials])
traces = [
go.Scatter(
x=[t.number for t in trials],
y=[t.value for t in trials],
mode="markers",
name=target_name,
),
go.Scatter(x=[t.number for t in trials],
y=best_values,
name="Best Value"),
]
else:
traces = [
go.Scatter(
x=[t.number for t in trials],
y=[target(t) for t in trials],
mode="markers",
name=target_name,
),
]
figure = go.Figure(data=traces, layout=layout)
return figure
def _get_pareto_front_2d(
study: MultiObjectiveStudy,
names: Optional[List[str]],
include_dominated_trials: bool = False,
axis_order: Optional[List[int]] = None,
) -> "go.Figure":
if names is None:
names = ["Objective 0", "Objective 1"]
elif len(names) != 2:
raise ValueError("The length of `names` is supposed to be 2.")
trials = study.get_pareto_front_trials()
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
point_colors = ["blue"] * len(trials)
if include_dominated_trials:
non_pareto_trials = _get_non_pareto_front_trials(study, trials)
point_colors += ["red"] * len(non_pareto_trials)
trials += non_pareto_trials
if axis_order is None:
axis_order = list(range(2))
else:
if len(axis_order) != 2:
raise ValueError(
f"Size of `axis_order` {axis_order}. Expect: 2, Actual: {len(axis_order)}."
)
if len(set(axis_order)) != 2:
raise ValueError(f"Elements of given `axis_order` {axis_order} are not unique!")
if max(axis_order) > 1:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} "
"higher than 1."
)
if min(axis_order) < 0:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} "
"lower than 0."
)
data = go.Scatter(
x=[t.values[axis_order[0]] for t in trials],
y=[t.values[axis_order[1]] for t in trials],
text=[_make_hovertext(t) for t in trials],
mode="markers",
hovertemplate="%{text}<extra></extra>",
marker={"color": point_colors},
)
layout = go.Layout(
title="Pareto-front Plot",
xaxis_title=names[axis_order[0]],
yaxis_title=names[axis_order[1]],
)
return go.Figure(data=data, layout=layout)
def _get_edf_plot(
studies: List[Study],
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(
title="Empirical Distribution Function Plot",
xaxis={"title": target_name},
yaxis={"title": "Cumulative Probability"},
)
if len(studies) == 0:
_logger.warning("There are no studies.")
return go.Figure(data=[], layout=layout)
all_trials = list(
itertools.chain.from_iterable(
(trial for trial in study.get_trials(deepcopy=False)
if trial.state == TrialState.COMPLETE) for study in studies))
if len(all_trials) == 0:
_logger.warning("There are no complete trials.")
return go.Figure(data=[], layout=layout)
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
min_x_value = min(target(trial) for trial in all_trials)
max_x_value = max(target(trial) for trial in all_trials)
x_values = np.linspace(min_x_value, max_x_value, 100)
traces = []
for study in studies:
values = np.asarray([
target(trial) for trial in study.get_trials(deepcopy=False)
if trial.state == TrialState.COMPLETE
])
y_values = np.sum(values[:, np.newaxis] <= x_values,
axis=0) / values.size
traces.append(
go.Scatter(x=x_values,
y=y_values,
name=study.study_name,
mode="lines"))
figure = go.Figure(data=traces, layout=layout)
figure.update_yaxes(range=[0, 1])
return figure
def _get_edf_plot(
studies: List[Study],
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(
title="Empirical Distribution Function Plot",
xaxis={"title": target_name},
yaxis={"title": "Cumulative Probability"},
)
if len(studies) == 0:
_logger.warning("There are no studies.")
return go.Figure(data=[], layout=layout)
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
target = _target
all_values: List[np.ndarray] = []
for study in studies:
trials = _filter_nonfinite(
study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target
)
values = np.array([target(trial) for trial in trials])
all_values.append(values)
if all(len(values) == 0 for values in all_values):
_logger.warning("There are no complete trials.")
return go.Figure(data=[], layout=layout)
min_x_value = np.min(np.concatenate(all_values))
max_x_value = np.max(np.concatenate(all_values))
x_values = np.linspace(min_x_value, max_x_value, 100)
traces = []
for values, study in zip(all_values, studies):
y_values = np.sum(values[:, np.newaxis] <= x_values, axis=0) / values.size
traces.append(go.Scatter(x=x_values, y=y_values, name=study.study_name, mode="lines"))
figure = go.Figure(data=traces, layout=layout)
figure.update_yaxes(range=[0, 1])
return figure
def _generate_slice_subplot(study: Study, trials: List[FrozenTrial], param: str) -> "Scatter":
return go.Scatter(
x=[t.params[param] for t in trials if param in t.params],
y=[t.value for t in trials if param in t.params],
mode="markers",
marker={
"line": {"width": 0.5, "color": "Grey",},
"color": [t.number for t in trials if param in t.params],
"colorscale": "Blues",
"colorbar": {
"title": "#Trials",
"x": 1.0, # Offset the colorbar position with a fixed width `xpad`.
"xpad": 40,
},
},
showlegend=False,
)
def _make_scatter_object(
n_targets: int,
axis_order: Sequence[int],
include_dominated_trials: bool,
trials_with_values: Optional[Sequence[Tuple[FrozenTrial,
Sequence[float]]]],
hovertemplate: str,
infeasible: bool = False,
dominated_trials: bool = False,
) -> Union["go.Scatter", "go.Scatter3d"]:
trials_with_values = trials_with_values or []
assert n_targets in (2, 3)
marker = _make_marker(
[trial for trial, _ in trials_with_values],
include_dominated_trials,
dominated_trials=dominated_trials,
infeasible=infeasible,
)
if n_targets == 2:
return go.Scatter(
x=[values[axis_order[0]] for _, values in trials_with_values],
y=[values[axis_order[1]] for _, values in trials_with_values],
text=[_make_hovertext(trial) for trial, _ in trials_with_values],
mode="markers",
hovertemplate=hovertemplate,
marker=marker,
showlegend=False,
)
else:
assert n_targets == 3
return go.Scatter3d(
x=[values[axis_order[0]] for _, values in trials_with_values],
y=[values[axis_order[1]] for _, values in trials_with_values],
z=[values[axis_order[2]] for _, values in trials_with_values],
text=[_make_hovertext(trial) for trial, _ in trials_with_values],
mode="markers",
hovertemplate=hovertemplate,
marker=marker,
showlegend=False,
)
def _generate_slice_subplot(subplot_info: _SliceSubplotInfo) -> "Scatter":
return go.Scatter(
x=subplot_info.x,
y=subplot_info.y,
mode="markers",
marker={
"line": {
"width": 0.5,
"color": "Grey"
},
"color": subplot_info.trial_numbers,
"colorscale": COLOR_SCALE,
"colorbar": {
"title": "Trial",
"x":
1.0, # Offset the colorbar position with a fixed width `xpad`.
"xpad": 40,
},
},
showlegend=False,
)
def _get_intermediate_plot(study: Study) -> "go.Figure":
layout = go.Layout(
title="Intermediate Values Plot",
xaxis={"title": "Step"},
yaxis={"title": "Intermediate Value"},
showlegend=False,
)
target_state = [TrialState.PRUNED, TrialState.COMPLETE, TrialState.RUNNING]
trials = [trial for trial in study.trials if trial.state in target_state]
if len(trials) == 0:
_logger.warning("Study instance does not contain trials.")
return go.Figure(data=[], layout=layout)
traces = []
for trial in trials:
if trial.intermediate_values:
sorted_intermediate_values = sorted(
trial.intermediate_values.items())
trace = go.Scatter(
x=tuple((x for x, _ in sorted_intermediate_values)),
y=tuple((y for _, y in sorted_intermediate_values)),
mode="lines+markers",
marker={"maxdisplayed": 10},
name="Trial{}".format(trial.number),
)
traces.append(trace)
if not traces:
_logger.warning(
"You need to set up the pruning feature to utilize `plot_intermediate_values()`"
)
return go.Figure(data=[], layout=layout)
figure = go.Figure(data=traces, layout=layout)
return figure
def _make_scatter_object_base(
n_dim: int,
trials: Sequence[FrozenTrial],
axis_order: List[int],
include_dominated_trials: bool,
hovertemplate: str,
infeasible: bool = False,
dominated_trials: bool = False,
) -> Union["go.Scatter", "go.Scatter3d"]:
assert n_dim in (2, 3)
marker = _make_marker(
trials,
include_dominated_trials,
dominated_trials=dominated_trials,
infeasible=infeasible,
)
if n_dim == 2:
return go.Scatter(
x=[t.values[axis_order[0]] for t in trials],
y=[t.values[axis_order[1]] for t in trials],
text=[_make_hovertext(t) for t in trials],
mode="markers",
hovertemplate=hovertemplate,
marker=marker,
showlegend=False,
)
else:
assert n_dim == 3
return go.Scatter3d(
x=[t.values[axis_order[0]] for t in trials],
y=[t.values[axis_order[1]] for t in trials],
z=[t.values[axis_order[2]] for t in trials],
text=[_make_hovertext(t) for t in trials],
mode="markers",
hovertemplate=hovertemplate,
marker=marker,
showlegend=False,
)
def _get_contour_plot(
study: Study,
params: Optional[List[str]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
layout = go.Layout(title="Contour Plot")
trials = _filter_nonfinite(study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, )),
target=target)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return go.Figure(data=[], layout=layout)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(all_params)
elif len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
return go.Figure(data=[], layout=layout)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError(
"Parameter {} does not exist in your study.".format(
input_p_name))
sorted_params = sorted(set(params))
padding_ratio = 0.05
param_values_range = {}
for p_name in sorted_params:
values = _get_param_values(trials, p_name)
min_value = min(values)
max_value = max(values)
if _is_log_scale(trials, p_name):
padding = (math.log10(max_value) -
math.log10(min_value)) * padding_ratio
min_value = math.pow(10, math.log10(min_value) - padding)
max_value = math.pow(10, math.log10(max_value) + padding)
elif _is_numerical(trials, p_name):
padding = (max_value - min_value) * padding_ratio
min_value = min_value - padding
max_value = max_value + padding
else:
# Plotly>=4.12.0 draws contours using the indices of categorical variables instead of
# raw values and the range should be updated based on the cardinality of categorical
# variables. See https://github.com/optuna/optuna/issues/1967.
if version.parse(plotly.__version__) >= version.parse("4.12.0"):
span = len(set(values)) - 1
padding = span * padding_ratio
min_value = -padding
max_value = span + padding
param_values_range[p_name] = (min_value, max_value)
reverse_scale = _is_reverse_scale(study, target)
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plots = _generate_contour_subplot(trials, x_param, y_param,
reverse_scale,
param_values_range, target,
target_name)
figure = go.Figure(data=sub_plots, layout=layout)
figure.update_xaxes(title_text=x_param,
range=param_values_range[x_param])
figure.update_yaxes(title_text=y_param,
range=param_values_range[y_param])
if not _is_numerical(trials, x_param):
figure.update_xaxes(type="category")
if not _is_numerical(trials, y_param):
figure.update_yaxes(type="category")
if _is_log_scale(trials, x_param):
log_range = [math.log10(p) for p in param_values_range[x_param]]
figure.update_xaxes(range=log_range, type="log")
if _is_log_scale(trials, y_param):
log_range = [math.log10(p) for p in param_values_range[y_param]]
figure.update_yaxes(range=log_range, type="log")
else:
figure = make_subplots(rows=len(sorted_params),
cols=len(sorted_params),
shared_xaxes=True,
shared_yaxes=True)
figure.update_layout(layout)
showscale = True # showscale option only needs to be specified once
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
if x_param == y_param:
figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1)
else:
sub_plots = _generate_contour_subplot(
trials,
x_param,
y_param,
reverse_scale,
param_values_range,
target,
target_name,
)
contour = sub_plots[0]
scatter = sub_plots[1]
contour.update(
showscale=showscale) # showscale's default is True
if showscale:
showscale = False
figure.add_trace(contour, row=y_i + 1, col=x_i + 1)
figure.add_trace(scatter, row=y_i + 1, col=x_i + 1)
figure.update_xaxes(range=param_values_range[x_param],
row=y_i + 1,
col=x_i + 1)
figure.update_yaxes(range=param_values_range[y_param],
row=y_i + 1,
col=x_i + 1)
if not _is_numerical(trials, x_param):
figure.update_xaxes(type="category",
row=y_i + 1,
col=x_i + 1)
if not _is_numerical(trials, y_param):
figure.update_yaxes(type="category",
row=y_i + 1,
col=x_i + 1)
if _is_log_scale(trials, x_param):
log_range = [
math.log10(p) for p in param_values_range[x_param]
]
figure.update_xaxes(range=log_range,
type="log",
row=y_i + 1,
col=x_i + 1)
if _is_log_scale(trials, y_param):
log_range = [
math.log10(p) for p in param_values_range[y_param]
]
figure.update_yaxes(range=log_range,
type="log",
row=y_i + 1,
col=x_i + 1)
if x_i == 0:
figure.update_yaxes(title_text=y_param,
row=y_i + 1,
col=x_i + 1)
if y_i == len(sorted_params) - 1:
figure.update_xaxes(title_text=x_param,
row=y_i + 1,
col=x_i + 1)
return figure
def _generate_contour_subplot(
trials: List[FrozenTrial],
x_param: str,
y_param: str,
reverse_scale: bool,
param_values_range: Optional[Dict[str, Tuple[float, float]]] = None,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> Tuple["Contour", "Scatter"]:
if param_values_range is None:
param_values_range = {}
x_indices = sorted(set(_get_param_values(trials, x_param)))
y_indices = sorted(set(_get_param_values(trials, y_param)))
if len(x_indices) < 2:
_logger.warning(
"Param {} unique value length is less than 2.".format(x_param))
return go.Contour(), go.Scatter()
if len(y_indices) < 2:
_logger.warning(
"Param {} unique value length is less than 2.".format(y_param))
return go.Contour(), go.Scatter()
# Padding to the plot for non-categorical params.
x_range = param_values_range[x_param]
if _is_numerical(trials, x_param):
x_indices = [x_range[0]] + x_indices + [x_range[1]]
y_range = param_values_range[y_param]
if _is_numerical(trials, y_param):
y_indices = [y_range[0]] + y_indices + [y_range[1]]
z = [[float("nan") for _ in range(len(x_indices))]
for _ in range(len(y_indices))]
x_values = []
y_values = []
for trial in trials:
if x_param not in trial.params or y_param not in trial.params:
continue
x_value = trial.params[x_param]
y_value = trial.params[y_param]
if not _is_numerical(trials, x_param):
x_value = str(x_value)
if not _is_numerical(trials, y_param):
y_value = str(y_value)
x_values.append(x_value)
y_values.append(y_value)
x_i = x_indices.index(x_value)
y_i = y_indices.index(y_value)
if target is None:
value = trial.value
else:
value = target(trial)
if isinstance(value, int):
value = float(value)
elif not isinstance(value, float):
raise ValueError(
f"Trial{trial.number} has COMPLETE state, but its target value is non-numeric."
)
z[y_i][x_i] = value
contour = go.Contour(
x=x_indices,
y=y_indices,
z=z,
colorbar={"title": target_name},
colorscale=COLOR_SCALE,
connectgaps=True,
contours_coloring="heatmap",
hoverinfo="none",
line_smoothing=1.3,
reversescale=reverse_scale,
)
scatter = go.Scatter(
x=x_values,
y=y_values,
marker={
"line": {
"width": 2.0,
"color": "Grey"
},
"color": "black"
},
mode="markers",
showlegend=False,
)
return (contour, scatter)
def _get_contour_plot(study: Study, params: Optional[List[str]] = None) -> "go.Figure":
layout = go.Layout(title="Contour Plot")
trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE]
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
return go.Figure(data=[], layout=layout)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(list(all_params))
elif len(params) <= 1:
_logger.warning("The length of params must be greater than 1.")
return go.Figure(data=[], layout=layout)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError("Parameter {} does not exist in your study.".format(input_p_name))
sorted_params = sorted(list(set(params)))
padding_ratio = 0.05
param_values_range = {}
update_category_axes = {}
for p_name in sorted_params:
values = _get_param_values(trials, p_name)
min_value = min(values)
max_value = max(values)
if _is_log_scale(trials, p_name):
padding = (math.log10(max_value) - math.log10(min_value)) * padding_ratio
min_value = math.pow(10, math.log10(min_value) - padding)
max_value = math.pow(10, math.log10(max_value) + padding)
elif _is_categorical(trials, p_name):
# For numeric values, plotly does not automatically plot as "category" type.
update_category_axes[p_name] = any([str(v).isnumeric() for v in set(values)])
else:
padding = (max_value - min_value) * padding_ratio
min_value = min_value - padding
max_value = max_value + padding
param_values_range[p_name] = (min_value, max_value)
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plots = _generate_contour_subplot(
trials, x_param, y_param, study.direction, param_values_range
)
figure = go.Figure(data=sub_plots, layout=layout)
figure.update_xaxes(title_text=x_param, range=param_values_range[x_param])
figure.update_yaxes(title_text=y_param, range=param_values_range[y_param])
if update_category_axes.get(x_param, False):
figure.update_xaxes(type="category")
if update_category_axes.get(y_param, False):
figure.update_yaxes(type="category")
if _is_log_scale(trials, x_param):
log_range = [math.log10(p) for p in param_values_range[x_param]]
figure.update_xaxes(range=log_range, type="log")
if _is_log_scale(trials, y_param):
log_range = [math.log10(p) for p in param_values_range[y_param]]
figure.update_yaxes(range=log_range, type="log")
else:
figure = make_subplots(
rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True
)
figure.update_layout(layout)
showscale = True # showscale option only needs to be specified once
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
if x_param == y_param:
figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1)
else:
sub_plots = _generate_contour_subplot(
trials, x_param, y_param, study.direction, param_values_range
)
contour = sub_plots[0]
scatter = sub_plots[1]
contour.update(showscale=showscale) # showscale's default is True
if showscale:
showscale = False
figure.add_trace(contour, row=y_i + 1, col=x_i + 1)
figure.add_trace(scatter, row=y_i + 1, col=x_i + 1)
figure.update_xaxes(range=param_values_range[x_param], row=y_i + 1, col=x_i + 1)
figure.update_yaxes(range=param_values_range[y_param], row=y_i + 1, col=x_i + 1)
if update_category_axes.get(x_param, False):
figure.update_xaxes(type="category", row=y_i + 1, col=x_i + 1)
if update_category_axes.get(y_param, False):
figure.update_yaxes(type="category", row=y_i + 1, col=x_i + 1)
if _is_log_scale(trials, x_param):
log_range = [math.log10(p) for p in param_values_range[x_param]]
figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1)
if _is_log_scale(trials, y_param):
log_range = [math.log10(p) for p in param_values_range[y_param]]
figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1)
if x_i == 0:
figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1)
if y_i == len(sorted_params) - 1:
figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1)
return figure
def _generate_contour_subplot(
trials: List[FrozenTrial],
x_param: str,
y_param: str,
direction: StudyDirection,
param_values_range: Optional[Dict[str, Tuple[float, float]]] = None,
) -> Tuple["Contour", "Scatter"]:
if param_values_range is None:
param_values_range = {}
x_indices = sorted(set(_get_param_values(trials, x_param)))
y_indices = sorted(set(_get_param_values(trials, y_param)))
if len(x_indices) < 2:
_logger.warning("Param {} unique value length is less than 2.".format(x_param))
return go.Contour(), go.Scatter()
if len(y_indices) < 2:
_logger.warning("Param {} unique value length is less than 2.".format(y_param))
return go.Contour(), go.Scatter()
# Padding to the plot for non-categorical params.
x_range = param_values_range[x_param]
if not _is_categorical(trials, x_param):
x_indices = [x_range[0]] + x_indices + [x_range[1]]
y_range = param_values_range[y_param]
if not _is_categorical(trials, y_param):
y_indices = [y_range[0]] + y_indices + [y_range[1]]
z = [[float("nan") for _ in range(len(x_indices))] for _ in range(len(y_indices))]
x_values = []
y_values = []
for trial in trials:
if x_param not in trial.params or y_param not in trial.params:
continue
x_value = trial.params[x_param]
y_value = trial.params[y_param]
if _is_categorical(trials, x_param):
x_value = str(x_value)
if _is_categorical(trials, y_param):
y_value = str(y_value)
x_values.append(x_value)
y_values.append(y_value)
x_i = x_indices.index(x_value)
y_i = y_indices.index(y_value)
if isinstance(trial.value, int):
value = float(trial.value)
elif isinstance(trial.value, float):
value = trial.value
else:
raise ValueError(
"Trial{} has COMPLETE state, but its value is non-numeric.".format(trial.number)
)
z[y_i][x_i] = value
# TODO(Yanase): Use reversescale argument to reverse colorscale if Plotly's bug is fixed.
# If contours_coloring='heatmap' is specified, reversescale argument of go.Contour does not
# work correctly. See https://github.com/pfnet/optuna/issues/606.
colorscale = plotly.colors.PLOTLY_SCALES["Blues"]
if direction == StudyDirection.MAXIMIZE:
colorscale = [[1 - t[0], t[1]] for t in colorscale]
colorscale.reverse()
contour = go.Contour(
x=x_indices,
y=y_indices,
z=z,
colorbar={"title": "Objective Value"},
colorscale=colorscale,
connectgaps=True,
contours_coloring="heatmap",
hoverinfo="none",
line_smoothing=1.3,
)
scatter = go.Scatter(
x=x_values,
y=y_values,
marker={"line": {"width": 0.5, "color": "Grey"}, "color": "black"},
mode="markers",
showlegend=False,
)
return (contour, scatter)
def _get_pareto_front_2d(
study: Study,
target_names: Optional[List[str]],
include_dominated_trials: bool = False,
axis_order: Optional[List[int]] = None,
) -> "go.Figure":
if target_names is None:
target_names = ["Objective 0", "Objective 1"]
elif len(target_names) != 2:
raise ValueError("The length of `target_names` is supposed to be 2.")
trials = study.best_trials
n_best_trials = len(trials)
if len(trials) == 0:
_logger.warning("Your study does not have any completed trials.")
if include_dominated_trials:
non_pareto_trials = _get_non_pareto_front_trials(study, trials)
trials += non_pareto_trials
if axis_order is None:
axis_order = list(range(2))
else:
if len(axis_order) != 2:
raise ValueError(
f"Size of `axis_order` {axis_order}. Expect: 2, Actual: {len(axis_order)}."
)
if len(set(axis_order)) != 2:
raise ValueError(f"Elements of given `axis_order` {axis_order} are not unique!")
if max(axis_order) > 1:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} "
"higher than 1."
)
if min(axis_order) < 0:
raise ValueError(
f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} "
"lower than 0."
)
data = [
go.Scatter(
x=[t.values[axis_order[0]] for t in trials[n_best_trials:]],
y=[t.values[axis_order[1]] for t in trials[n_best_trials:]],
text=[_make_hovertext(t) for t in trials[n_best_trials:]],
mode="markers",
hovertemplate="%{text}<extra>Trial</extra>",
marker={
"line": {"width": 0.5, "color": "Grey"},
"color": [t.number for t in trials[n_best_trials:]],
"colorscale": "Blues",
"colorbar": {
"title": "#Trials",
},
},
showlegend=False,
),
go.Scatter(
x=[t.values[axis_order[0]] for t in trials[:n_best_trials]],
y=[t.values[axis_order[1]] for t in trials[:n_best_trials]],
text=[_make_hovertext(t) for t in trials[:n_best_trials]],
mode="markers",
hovertemplate="%{text}<extra>Best Trial</extra>",
marker={
"line": {"width": 0.5, "color": "Grey"},
"color": [t.number for t in trials[:n_best_trials]],
"colorscale": "Reds",
"colorbar": {
"title": "#Best trials",
"x": 1.1 if include_dominated_trials else 1,
"xpad": 40,
},
},
showlegend=False,
),
]
layout = go.Layout(
title="Pareto-front Plot",
xaxis_title=target_names[axis_order[0]],
yaxis_title=target_names[axis_order[1]],
)
return go.Figure(data=data, layout=layout)
def plot_edf(
study: Union[Study, Sequence[Study]],
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
) -> "go.Figure":
"""Plot the objective value EDF (empirical distribution function) of a study.
Note that only the complete trials are considered when plotting the EDF.
.. note::
EDF is useful to analyze and improve search spaces.
For instance, you can see a practical use case of EDF in the paper
`Designing Network Design Spaces <https://arxiv.org/abs/2003.13678>`_.
.. note::
The plotted EDF assumes that the value of the objective function is in
accordance with the uniform distribution over the objective space.
Example:
The following code snippet shows how to plot EDF.
.. plotly::
import math
import optuna
def ackley(x, y):
a = 20 * math.exp(-0.2 * math.sqrt(0.5 * (x ** 2 + y ** 2)))
b = math.exp(0.5 * (math.cos(2 * math.pi * x) + math.cos(2 * math.pi * y)))
return -a - b + math.e + 20
def objective(trial, low, high):
x = trial.suggest_float("x", low, high)
y = trial.suggest_float("y", low, high)
return ackley(x, y)
sampler = optuna.samplers.RandomSampler(seed=10)
# Widest search space.
study0 = optuna.create_study(study_name="x=[0,5), y=[0,5)", sampler=sampler)
study0.optimize(lambda t: objective(t, 0, 5), n_trials=500)
# Narrower search space.
study1 = optuna.create_study(study_name="x=[0,4), y=[0,4)", sampler=sampler)
study1.optimize(lambda t: objective(t, 0, 4), n_trials=500)
# Narrowest search space but it doesn't include the global optimum point.
study2 = optuna.create_study(study_name="x=[1,3), y=[1,3)", sampler=sampler)
study2.optimize(lambda t: objective(t, 1, 3), n_trials=500)
fig = optuna.visualization.plot_edf([study0, study1, study2])
fig.show()
Args:
study:
A target :class:`~optuna.study.Study` object.
You can pass multiple studies if you want to compare those EDFs.
target:
A function to specify the value to display. If it is :obj:`None` and ``study`` is being
used for single-objective optimization, the objective values are plotted.
.. note::
Specify this argument if ``study`` is being used for multi-objective optimization.
target_name:
Target's name to display on the axis label.
Returns:
A :class:`plotly.graph_objs.Figure` object.
"""
_imports.check()
layout = go.Layout(
title="Empirical Distribution Function Plot",
xaxis={"title": target_name},
yaxis={"title": "Cumulative Probability"},
)
info = _get_edf_info(study, target, target_name)
edf_lines = info.lines
if len(edf_lines) == 0:
return go.Figure(data=[], layout=layout)
traces = []
for study_name, y_values in edf_lines:
traces.append(
go.Scatter(x=info.x_values,
y=y_values,
name=study_name,
mode="lines"))
figure = go.Figure(data=traces, layout=layout)
figure.update_yaxes(range=[0, 1])
return figure
def _get_optimization_histories_with_error_bar(
studies: List[Study],
target: Optional[Callable[[FrozenTrial], float]],
target_name: str,
layout: "go.Layout",
) -> "go.Figure":
max_trial_number = np.max(
[
trial.number
for study in studies
for trial in study.get_trials(states=(TrialState.COMPLETE,))
]
)
_target: Callable[[FrozenTrial], float]
if target is None:
def _target(t: FrozenTrial) -> float:
return cast(float, t.value)
else:
_target = target
target_values: List[List[float]] = [[] for _ in range(max_trial_number + 2)]
for study in studies:
trials = study.get_trials(states=(TrialState.COMPLETE,))
for t in trials:
target_values[t.number].append(_target(t))
mean_of_target_values = [np.mean(v) if len(v) > 0 else None for v in target_values]
std_of_target_values = [np.std(v) if len(v) > 0 else None for v in target_values]
trial_numbers = np.arange(max_trial_number + 2)[[v is not None for v in mean_of_target_values]]
means = np.asarray(mean_of_target_values)[trial_numbers]
stds = np.asarray(std_of_target_values)[trial_numbers]
traces = [
go.Scatter(
x=trial_numbers,
y=means,
error_y={
"type": "data",
"array": stds,
"visible": True,
},
mode="markers",
name=target_name,
)
]
if target is None:
best_values: List[List[float]] = [[] for _ in range(max_trial_number + 2)]
for study in studies:
trials = study.get_trials(states=(TrialState.COMPLETE,))
if study.direction == StudyDirection.MINIMIZE:
best_vs = np.minimum.accumulate([cast(float, t.value) for t in trials])
else:
best_vs = np.maximum.accumulate([cast(float, t.value) for t in trials])
for i, t in enumerate(trials):
best_values[t.number].append(best_vs[i])
mean_of_best_values = [np.mean(v) if len(v) > 0 else None for v in best_values]
std_of_best_values = [np.std(v) if len(v) > 0 else None for v in best_values]
means = np.asarray(mean_of_best_values)[trial_numbers]
stds = np.asarray(std_of_best_values)[trial_numbers]
traces.append(go.Scatter(x=trial_numbers, y=means, name="Best Value"))
traces.append(
go.Scatter(
x=trial_numbers,
y=means + stds,
mode="lines",
line=dict(width=0.01),
showlegend=False,
)
)
traces.append(
go.Scatter(
x=trial_numbers,
y=means - stds,
mode="none",
showlegend=False,
fill="tonexty",
fillcolor="rgba(255,0,0,0.2)",
)
)
figure = go.Figure(data=traces, layout=layout)
return figure
def _get_optimization_history_plot(
info_list: List[_OptimizationHistoryInfo],
target_name: str,
) -> "go.Figure":
layout = go.Layout(
title="Optimization History Plot",
xaxis={"title": "Trial"},
yaxis={"title": target_name},
)
traces = []
for trial_numbers, values_info, best_values_info in info_list:
if values_info.stds is None:
error_y = None
else:
error_y = {
"type": "data",
"array": values_info.stds,
"visible": True
}
traces.append(
go.Scatter(
x=trial_numbers,
y=values_info.values,
error_y=error_y,
mode="markers",
name=values_info.label_name,
))
if best_values_info is not None:
traces.append(
go.Scatter(
x=trial_numbers,
y=best_values_info.values,
name=best_values_info.label_name,
))
if best_values_info.stds is not None:
upper = np.array(best_values_info.values) + np.array(
best_values_info.stds)
traces.append(
go.Scatter(
x=trial_numbers,
y=upper,
mode="lines",
line=dict(width=0.01),
showlegend=False,
))
lower = np.array(best_values_info.values) - np.array(
best_values_info.stds)
traces.append(
go.Scatter(
x=trial_numbers,
y=lower,
mode="none",
showlegend=False,
fill="tonexty",
fillcolor="rgba(255,0,0,0.2)",
))
return go.Figure(data=traces, layout=layout)
def _get_contour_plot(info: _ContourInfo) -> "go.Figure":
layout = go.Layout(title="Contour Plot")
sorted_params = info.sorted_params
sub_plot_infos = info.sub_plot_infos
reverse_scale = info.reverse_scale
target_name = info.target_name
if len(sorted_params) <= 1:
return go.Figure(data=[], layout=layout)
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plot_info = sub_plot_infos[0][0]
sub_plots = _get_contour_subplot(sub_plot_info, reverse_scale, target_name)
figure = go.Figure(data=sub_plots, layout=layout)
figure.update_xaxes(title_text=x_param, range=sub_plot_info.xaxis.range)
figure.update_yaxes(title_text=y_param, range=sub_plot_info.yaxis.range)
if sub_plot_info.xaxis.is_cat:
figure.update_xaxes(type="category")
if sub_plot_info.yaxis.is_cat:
figure.update_yaxes(type="category")
if sub_plot_info.xaxis.is_log:
log_range = [math.log10(p) for p in sub_plot_info.xaxis.range]
figure.update_xaxes(range=log_range, type="log")
if sub_plot_info.yaxis.is_log:
log_range = [math.log10(p) for p in sub_plot_info.yaxis.range]
figure.update_yaxes(range=log_range, type="log")
else:
figure = make_subplots(
rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True
)
figure.update_layout(layout)
showscale = True # showscale option only needs to be specified once.
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
if x_param == y_param:
figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1)
else:
sub_plots = _get_contour_subplot(
sub_plot_infos[y_i][x_i], reverse_scale, target_name
)
contour = sub_plots[0]
scatter = sub_plots[1]
contour.update(showscale=showscale) # showscale's default is True.
if showscale:
showscale = False
figure.add_trace(contour, row=y_i + 1, col=x_i + 1)
figure.add_trace(scatter, row=y_i + 1, col=x_i + 1)
xaxis = sub_plot_infos[y_i][x_i].xaxis
yaxis = sub_plot_infos[y_i][x_i].yaxis
figure.update_xaxes(range=xaxis.range, row=y_i + 1, col=x_i + 1)
figure.update_yaxes(range=yaxis.range, row=y_i + 1, col=x_i + 1)
if xaxis.is_cat:
figure.update_xaxes(type="category", row=y_i + 1, col=x_i + 1)
if yaxis.is_cat:
figure.update_yaxes(type="category", row=y_i + 1, col=x_i + 1)
if xaxis.is_log:
log_range = [math.log10(p) for p in xaxis.range]
figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1)
if yaxis.is_log:
log_range = [math.log10(p) for p in yaxis.range]
figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1)
if x_i == 0:
figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1)
if y_i == len(sorted_params) - 1:
figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1)
return figure
def _generate_contour_subplot(
trials: List[FrozenTrial], x_param: str, y_param: str,
direction: StudyDirection) -> Tuple["Contour", "Scatter"]:
x_indices = sorted(
list({t.params[x_param]
for t in trials if x_param in t.params}))
y_indices = sorted(
list({t.params[y_param]
for t in trials if y_param in t.params}))
if len(x_indices) < 2:
_logger.warning(
"Param {} unique value length is less than 2.".format(x_param))
return go.Contour(), go.Scatter()
if len(y_indices) < 2:
_logger.warning(
"Param {} unique value length is less than 2.".format(y_param))
return go.Contour(), go.Scatter()
z = [[float("nan") for _ in range(len(x_indices))]
for _ in range(len(y_indices))]
x_values = []
y_values = []
for trial in trials:
if x_param not in trial.params or y_param not in trial.params:
continue
x_values.append(trial.params[x_param])
y_values.append(trial.params[y_param])
x_i = x_indices.index(trial.params[x_param])
y_i = y_indices.index(trial.params[y_param])
if isinstance(trial.value, int):
value = float(trial.value)
elif isinstance(trial.value, float):
value = trial.value
else:
raise ValueError(
"Trial{} has COMPLETE state, but its value is non-numeric.".
format(trial.number))
z[y_i][x_i] = value
# TODO(Yanase): Use reversescale argument to reverse colorscale if Plotly's bug is fixed.
# If contours_coloring='heatmap' is specified, reversesecale argument of go.Contour does not
# work correctly. See https://github.com/pfnet/optuna/issues/606.
colorscale = plotly.colors.PLOTLY_SCALES["Blues"]
if direction == StudyDirection.MINIMIZE:
colorscale = [[1 - t[0], t[1]] for t in colorscale]
colorscale.reverse()
contour = go.Contour(
x=x_indices,
y=y_indices,
z=z,
colorbar={"title": "Objective Value"},
colorscale=colorscale,
connectgaps=True,
contours_coloring="heatmap",
hoverinfo="none",
line_smoothing=1.3,
)
scatter = go.Scatter(x=x_values,
y=y_values,
marker={"color": "black"},
mode="markers",
showlegend=False)
return (contour, scatter)
