def _get_optimization_history_plot(study: Study) -> "go.Figure":
layout = go.Layout(
title="Optimization History Plot",
xaxis={"title": "#Trials"},
yaxis={"title": "Objective Value"},
)
trials = [t for t in study.trials if t.state == TrialState.COMPLETE]
if len(trials) == 0:
logger.warning("Study instance does not contain trials.")
return go.Figure(data=[], layout=layout)
best_values = [float("inf")] if study.direction == StudyDirection.MINIMIZE else [-float("inf")]
comp = min if study.direction == StudyDirection.MINIMIZE else max
for trial in trials:
trial_value = trial.value
assert trial_value is not None # For mypy
best_values.append(comp(best_values[-1], trial_value))
best_values.pop(0)
traces = [
go.Scatter(
x=[t.number for t in trials],
y=[t.value for t in trials],
mode="markers",
name="Objective Value",
),
go.Scatter(x=[t.number for t in trials], y=best_values, name="Best Value"),
]
figure = go.Figure(data=traces, layout=layout)
return figure
def _generate_contour_subplot(trials, x_param, y_param, direction):
# type: (List[FrozenTrial], str, str, StudyDirection) -> Tuple[Contour, Scatter]
x_indices = sorted(list({t.params[x_param] for t in trials if x_param in t.params}))
y_indices = sorted(list({t.params[y_param] for t in trials if y_param in t.params}))
if len(x_indices) < 2:
logger.warning('Param {} unique value length is less than 2.'.format(x_param))
return go.Contour(), go.Scatter()
if len(y_indices) < 2:
logger.warning('Param {} unique value length is less than 2.'.format(y_param))
return go.Contour(), go.Scatter()
z = [[float('nan') for _ in range(len(x_indices))] for _ in range(len(y_indices))]
x_values = []
y_values = []
for trial in trials:
if x_param not in trial.params or y_param not in trial.params:
continue
x_values.append(trial.params[x_param])
y_values.append(trial.params[y_param])
x_i = x_indices.index(trial.params[x_param])
y_i = y_indices.index(trial.params[y_param])
if isinstance(trial.value, int):
value = float(trial.value)
elif isinstance(trial.value, float):
value = trial.value
else:
raise ValueError(
'Trial{} has COMPLETE state, but its value is non-numeric.'.format(trial.number))
z[y_i][x_i] = value
# TODO(Yanase): Use reversescale argument to reverse colorscale if Plotly's bug is fixed.
# If contours_coloring='heatmap' is specified, reversesecale argument of go.Contour does not
# work correctly. See https://github.com/pfnet/optuna/issues/606.
colorscale = plotly.colors.PLOTLY_SCALES['Blues']
if direction == StudyDirection.MINIMIZE:
colorscale = [[1 - t[0], t[1]] for t in colorscale]
colorscale.reverse()
contour = go.Contour(
x=x_indices, y=y_indices, z=z,
colorbar={'title': 'Objective Value'},
colorscale=colorscale,
connectgaps=True,
contours_coloring='heatmap',
hoverinfo='none',
line_smoothing=1.3,
)
scatter = go.Scatter(
x=x_values,
y=y_values,
marker={'color': 'black'},
mode='markers',
showlegend=False
)
return (contour, scatter)
def _get_intermediate_plot(study):
# type: (Study) -> go.Figure
layout = go.Layout(title='Intermediate Values Plot',
xaxis={'title': 'Step'},
yaxis={'title': 'Intermediate Value'},
showlegend=False)
target_state = [TrialState.PRUNED, TrialState.COMPLETE, TrialState.RUNNING]
trials = [trial for trial in study.trials if trial.state in target_state]
if len(trials) == 0:
logger.warning('Study instance does not contain trials.')
return go.Figure(data=[], layout=layout)
traces = []
for trial in trials:
if trial.intermediate_values:
trace = go.Scatter(x=tuple(trial.intermediate_values.keys()),
y=tuple(trial.intermediate_values.values()),
mode='lines+markers',
marker={'maxdisplayed': 10},
name='Trial{}'.format(trial.number))
traces.append(trace)
if not traces:
logger.warning(
'You need to set up the pruning feature to utilize `plot_intermediate_values()`'
)
return go.Figure(data=[], layout=layout)
figure = go.Figure(data=traces, layout=layout)
return figure
def _get_optimization_history_plot(study):
# type: (Study) -> Figure
layout = go.Layout(
title='Optimization History Plot',
xaxis={'title': '#Trials'},
yaxis={'title': 'Objective Value'},
)
trials = [t for t in study.trials if t.state == TrialState.COMPLETE]
if len(trials) == 0:
logger.warning('Study instance does not contain trials.')
return go.Figure(data=[], layout=layout)
best_values = [
float('inf')
] if study.direction == StudyDirection.MINIMIZE else [-float('inf')]
comp = min if study.direction == StudyDirection.MINIMIZE else max
for trial in trials:
trial_value = trial.value
assert trial_value is not None # For mypy
best_values.append(comp(best_values[-1], trial_value))
best_values.pop(0)
traces = [
go.Scatter(x=[t.number for t in trials],
y=[t.value for t in trials],
mode='markers',
name='Objective Value'),
go.Scatter(x=[t.number for t in trials],
y=best_values,
name='Best Value')
]
figure = go.Figure(data=traces, layout=layout)
return figure
def _generate_slice_subplot(study, trials, param):
# type: (Study, List[FrozenTrial], str) -> Scatter
return go.Scatter(
x=[t.params[param] for t in trials if param in t.params],
y=[t.value for t in trials if param in t.params],
mode="markers",
marker={
"line": {"width": 0.5, "color": "Grey",},
"color": [t.number for t in trials if param in t.params],
"colorscale": "Blues",
"colorbar": {
"title": "#Trials",
"x": 1.0, # Offset the colorbar position with a fixed width `xpad`.
"xpad": 40,
},
},
showlegend=False,
)
def _get_intermediate_plot(study):
# type: (Study) -> go.Figure
layout = go.Layout(
title="Intermediate Values Plot",
xaxis={"title": "Step"},
yaxis={"title": "Intermediate Value"},
showlegend=False,
)
target_state = [TrialState.PRUNED, TrialState.COMPLETE, TrialState.RUNNING]
trials = [trial for trial in study.trials if trial.state in target_state]
if len(trials) == 0:
logger.warning("Study instance does not contain trials.")
return go.Figure(data=[], layout=layout)
traces = []
for trial in trials:
if trial.intermediate_values:
sorted_intermediate_values = sorted(
trial.intermediate_values.items())
trace = go.Scatter(
x=tuple((x for x, _ in sorted_intermediate_values)),
y=tuple((y for _, y in sorted_intermediate_values)),
mode="lines+markers",
marker={"maxdisplayed": 10},
name="Trial{}".format(trial.number),
)
traces.append(trace)
if not traces:
logger.warning(
"You need to set up the pruning feature to utilize `plot_intermediate_values()`"
)
return go.Figure(data=[], layout=layout)
figure = go.Figure(data=traces, layout=layout)
return figure
def _generate_slice_subplot(study, trials, param):
# type: (Study, List[FrozenTrial], str) -> Scatter
return go.Scatter(
x=[t.params[param] for t in trials if param in t.params],
y=[t.value for t in trials if param in t.params],
mode='markers',
marker={
'line': {
'width': 0.5,
'color': 'Grey',
},
'color': [t.number for t in trials if param in t.params],
'colorscale': 'Blues',
'colorbar': {
'title': '#Trials',
'x':
1.0, # Offset the colorbar position with a fixed width `xpad`.
'xpad': 40,
}
},
showlegend=False,
)
def _get_contour_plot(study, params=None):
# type: (Study, Optional[List[str]]) -> Figure
layout = go.Layout(
title='Contour Plot',
)
trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE]
if len(trials) == 0:
logger.warning('Your study does not have any completed trials.')
return go.Figure(data=[], layout=layout)
all_params = {p_name for t in trials for p_name in t.params.keys()}
if params is None:
sorted_params = sorted(list(all_params))
elif len(params) <= 1:
logger.warning('The length of params must be greater than 1.')
return go.Figure(data=[], layout=layout)
else:
for input_p_name in params:
if input_p_name not in all_params:
raise ValueError('Parameter {} does not exist in your study.'.format(input_p_name))
sorted_params = sorted(list(set(params)))
param_values_range = {}
for p_name in sorted_params:
values = [t.params[p_name] for t in trials if p_name in t.params]
param_values_range[p_name] = (min(values), max(values))
if len(sorted_params) == 2:
x_param = sorted_params[0]
y_param = sorted_params[1]
sub_plots = _generate_contour_subplot(
trials, x_param, y_param, study.direction)
figure = go.Figure(data=sub_plots)
figure.update_xaxes(title_text=x_param, range=param_values_range[x_param])
figure.update_yaxes(title_text=y_param, range=param_values_range[y_param])
if _is_log_scale(trials, x_param):
log_range = [math.log10(p) for p in param_values_range[x_param]]
figure.update_xaxes(range=log_range, type='log')
if _is_log_scale(trials, y_param):
log_range = [math.log10(p) for p in param_values_range[y_param]]
figure.update_yaxes(range=log_range, type='log')
else:
figure = make_subplots(rows=len(sorted_params),
cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True)
showscale = True # showscale option only needs to be specified once
for x_i, x_param in enumerate(sorted_params):
for y_i, y_param in enumerate(sorted_params):
if x_param == y_param:
figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1)
else:
sub_plots = _generate_contour_subplot(
trials, x_param, y_param, study.direction)
contour = sub_plots[0]
scatter = sub_plots[1]
contour.update(showscale=showscale) # showscale's default is True
if showscale:
showscale = False
figure.add_trace(contour, row=y_i + 1, col=x_i + 1)
figure.add_trace(scatter, row=y_i + 1, col=x_i + 1)
figure.update_xaxes(range=param_values_range[x_param],
row=y_i + 1, col=x_i + 1)
figure.update_yaxes(range=param_values_range[y_param],
row=y_i + 1, col=x_i + 1)
if _is_log_scale(trials, x_param):
log_range = [math.log10(p) for p in param_values_range[x_param]]
figure.update_xaxes(range=log_range, type='log', row=y_i + 1, col=x_i + 1)
if _is_log_scale(trials, y_param):
log_range = [math.log10(p) for p in param_values_range[y_param]]
figure.update_yaxes(range=log_range, type='log', row=y_i + 1, col=x_i + 1)
if x_i == 0:
figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1)
if y_i == len(sorted_params) - 1:
figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1)
return figure
