def _get_optimization_history_plot( study: Study, target: Optional[Callable[[FrozenTrial], float]], target_name: str, ) -> "go.Figure": layout = go.Layout( title="Optimization History Plot", xaxis={"title": "#Trials"}, yaxis={"title": target_name}, ) trials = [t for t in study.trials if t.state == TrialState.COMPLETE] if len(trials) == 0: _logger.warning("Study instance does not contain trials.") return go.Figure(data=[], layout=layout) if target is None: if study.direction == StudyDirection.MINIMIZE: best_values = [float("inf")] else: best_values = [-float("inf")] comp = min if study.direction == StudyDirection.MINIMIZE else max for trial in trials: trial_value = trial.value assert trial_value is not None # For mypy best_values.append(comp(best_values[-1], trial_value)) best_values.pop(0) traces = [ go.Scatter( x=[t.number for t in trials], y=[t.value for t in trials], mode="markers", name=target_name, ), go.Scatter(x=[t.number for t in trials], y=best_values, name="Best Value"), ] else: traces = [ go.Scatter( x=[t.number for t in trials], y=[target(t) for t in trials], mode="markers", name=target_name, ), ] figure = go.Figure(data=traces, layout=layout) return figure
def _get_optimization_histories( studies: List[Study], target: Optional[Callable[[FrozenTrial], float]], target_name: str, layout: "go.Layout", ) -> "go.Figure": traces = [] for study in studies: trials = study.get_trials(states=(TrialState.COMPLETE,)) if target is None: if study.direction == StudyDirection.MINIMIZE: best_values = np.minimum.accumulate([cast(float, t.value) for t in trials]) else: best_values = np.maximum.accumulate([cast(float, t.value) for t in trials]) traces.append( go.Scatter( x=[t.number for t in trials], y=[t.value for t in trials], mode="markers", name=target_name if len(studies) == 1 else f"{target_name} of {study.study_name}", ) ) traces.append( go.Scatter( x=[t.number for t in trials], y=best_values, name="Best Value" if len(studies) == 1 else f"Best Value of {study.study_name}", ) ) else: traces.append( go.Scatter( x=[t.number for t in trials], y=[target(t) for t in trials], mode="markers", name=target_name if len(studies) == 1 else f"{target_name} of {study.study_name}", ) ) figure = go.Figure(data=traces, layout=layout) figure.update_layout(width=1000, height=400) return figure
def _get_contour_subplot( info: _SubContourInfo, reverse_scale: bool, target_name: str = "Objective Value", ) -> Tuple["Contour", "Scatter"]: x_indices = info.xaxis.indices y_indices = info.yaxis.indices x_values = [] y_values = [] for x_value, y_value in zip(info.xaxis.values, info.yaxis.values): if x_value is not None and y_value is not None: x_values.append(x_value) y_values.append(y_value) z_values = [[float("nan") for _ in range(len(info.xaxis.indices))] for _ in range(len(info.yaxis.indices))] for (x_i, y_i), z_value in info.z_values.items(): z_values[y_i][x_i] = z_value if len(x_indices) < 2 or len(y_indices) < 2: return go.Contour(), go.Scatter() contour = go.Contour( x=x_indices, y=y_indices, z=z_values, colorbar={"title": target_name}, colorscale=COLOR_SCALE, connectgaps=True, contours_coloring="heatmap", hoverinfo="none", line_smoothing=1.3, reversescale=reverse_scale, ) scatter = go.Scatter( x=x_values, y=y_values, marker={ "line": { "width": 2.0, "color": "Grey" }, "color": "black" }, mode="markers", showlegend=False, ) return contour, scatter
def _get_intermediate_plot(study: Study) -> "go.Figure": layout = go.Layout( title="Intermediate Values Plot", xaxis={"title": "Step"}, yaxis={"title": "Intermediate Value"}, showlegend=False, ) info = _get_intermediate_plot_info(study) trial_infos = info.trial_infos if len(trial_infos) == 0: return go.Figure(data=[], layout=layout) traces = [ go.Scatter( x=tuple((x for x, _ in tinfo.sorted_intermediate_values)), y=tuple((y for _, y in tinfo.sorted_intermediate_values)), mode="lines+markers", marker={"maxdisplayed": 10}, name="Trial{}".format(tinfo.trial_number), ) for tinfo in trial_infos ] return go.Figure(data=traces, layout=layout)
def _get_pareto_front_2d( study: MultiObjectiveStudy, names: Optional[List[str]], include_dominated_trials: bool = False) -> "go.Figure": if names is None: names = ["Objective 0", "Objective 1"] elif len(names) != 2: raise ValueError("The length of `names` is supposed to be 2.") trials = study.get_pareto_front_trials() if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") point_colors = ["blue"] * len(trials) if include_dominated_trials: non_pareto_trials = _get_non_pareto_front_trials(study, trials) point_colors += ["red"] * len(non_pareto_trials) trials += non_pareto_trials data = go.Scatter( x=[t.values[0] for t in trials], y=[t.values[1] for t in trials], text=[_make_hovertext(t) for t in trials], mode="markers", hovertemplate="%{text}<extra></extra>", marker={"color": point_colors}, ) layout = go.Layout(title="Pareto-front Plot", xaxis_title=names[0], yaxis_title=names[1]) return go.Figure(data=data, layout=layout)
def _generate_slice_subplot( trials: List[FrozenTrial], param: str, target: Optional[Callable[[FrozenTrial], float]], ) -> "Scatter": if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target return go.Scatter( x=[t.params[param] for t in trials if param in t.params], y=[target(t) for t in trials if param in t.params], mode="markers", marker={ "line": { "width": 0.5, "color": "Grey" }, "color": [t.number for t in trials if param in t.params], "colorscale": COLOR_SCALE, "colorbar": { "title": "Trial", "x": 1.0, # Offset the colorbar position with a fixed width `xpad`. "xpad": 40, }, }, showlegend=False, )
def _get_optimization_history_plot( study: Study, target: Optional[Callable[[FrozenTrial], float]], target_name: str, ) -> "go.Figure": layout = go.Layout( title="Optimization History Plot", xaxis={"title": "#Trials"}, yaxis={"title": target_name}, ) trials = [t for t in study.trials if t.state == TrialState.COMPLETE] if len(trials) == 0: _logger.warning("Study instance does not contain trials.") return go.Figure(data=[], layout=layout) if target is None: if study.direction == StudyDirection.MINIMIZE: best_values = np.minimum.accumulate([t.value for t in trials]) else: best_values = np.maximum.accumulate([t.value for t in trials]) traces = [ go.Scatter( x=[t.number for t in trials], y=[t.value for t in trials], mode="markers", name=target_name, ), go.Scatter(x=[t.number for t in trials], y=best_values, name="Best Value"), ] else: traces = [ go.Scatter( x=[t.number for t in trials], y=[target(t) for t in trials], mode="markers", name=target_name, ), ] figure = go.Figure(data=traces, layout=layout) return figure
def _get_pareto_front_2d( study: MultiObjectiveStudy, names: Optional[List[str]], include_dominated_trials: bool = False, axis_order: Optional[List[int]] = None, ) -> "go.Figure": if names is None: names = ["Objective 0", "Objective 1"] elif len(names) != 2: raise ValueError("The length of `names` is supposed to be 2.") trials = study.get_pareto_front_trials() if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") point_colors = ["blue"] * len(trials) if include_dominated_trials: non_pareto_trials = _get_non_pareto_front_trials(study, trials) point_colors += ["red"] * len(non_pareto_trials) trials += non_pareto_trials if axis_order is None: axis_order = list(range(2)) else: if len(axis_order) != 2: raise ValueError( f"Size of `axis_order` {axis_order}. Expect: 2, Actual: {len(axis_order)}." ) if len(set(axis_order)) != 2: raise ValueError(f"Elements of given `axis_order` {axis_order} are not unique!") if max(axis_order) > 1: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} " "higher than 1." ) if min(axis_order) < 0: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} " "lower than 0." ) data = go.Scatter( x=[t.values[axis_order[0]] for t in trials], y=[t.values[axis_order[1]] for t in trials], text=[_make_hovertext(t) for t in trials], mode="markers", hovertemplate="%{text}<extra></extra>", marker={"color": point_colors}, ) layout = go.Layout( title="Pareto-front Plot", xaxis_title=names[axis_order[0]], yaxis_title=names[axis_order[1]], ) return go.Figure(data=data, layout=layout)
def _get_edf_plot( studies: List[Study], target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout( title="Empirical Distribution Function Plot", xaxis={"title": target_name}, yaxis={"title": "Cumulative Probability"}, ) if len(studies) == 0: _logger.warning("There are no studies.") return go.Figure(data=[], layout=layout) all_trials = list( itertools.chain.from_iterable( (trial for trial in study.get_trials(deepcopy=False) if trial.state == TrialState.COMPLETE) for study in studies)) if len(all_trials) == 0: _logger.warning("There are no complete trials.") return go.Figure(data=[], layout=layout) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target min_x_value = min(target(trial) for trial in all_trials) max_x_value = max(target(trial) for trial in all_trials) x_values = np.linspace(min_x_value, max_x_value, 100) traces = [] for study in studies: values = np.asarray([ target(trial) for trial in study.get_trials(deepcopy=False) if trial.state == TrialState.COMPLETE ]) y_values = np.sum(values[:, np.newaxis] <= x_values, axis=0) / values.size traces.append( go.Scatter(x=x_values, y=y_values, name=study.study_name, mode="lines")) figure = go.Figure(data=traces, layout=layout) figure.update_yaxes(range=[0, 1]) return figure
def _get_edf_plot( studies: List[Study], target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout( title="Empirical Distribution Function Plot", xaxis={"title": target_name}, yaxis={"title": "Cumulative Probability"}, ) if len(studies) == 0: _logger.warning("There are no studies.") return go.Figure(data=[], layout=layout) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target all_values: List[np.ndarray] = [] for study in studies: trials = _filter_nonfinite( study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target ) values = np.array([target(trial) for trial in trials]) all_values.append(values) if all(len(values) == 0 for values in all_values): _logger.warning("There are no complete trials.") return go.Figure(data=[], layout=layout) min_x_value = np.min(np.concatenate(all_values)) max_x_value = np.max(np.concatenate(all_values)) x_values = np.linspace(min_x_value, max_x_value, 100) traces = [] for values, study in zip(all_values, studies): y_values = np.sum(values[:, np.newaxis] <= x_values, axis=0) / values.size traces.append(go.Scatter(x=x_values, y=y_values, name=study.study_name, mode="lines")) figure = go.Figure(data=traces, layout=layout) figure.update_yaxes(range=[0, 1]) return figure
def _generate_slice_subplot(study: Study, trials: List[FrozenTrial], param: str) -> "Scatter": return go.Scatter( x=[t.params[param] for t in trials if param in t.params], y=[t.value for t in trials if param in t.params], mode="markers", marker={ "line": {"width": 0.5, "color": "Grey",}, "color": [t.number for t in trials if param in t.params], "colorscale": "Blues", "colorbar": { "title": "#Trials", "x": 1.0, # Offset the colorbar position with a fixed width `xpad`. "xpad": 40, }, }, showlegend=False, )
def _make_scatter_object( n_targets: int, axis_order: Sequence[int], include_dominated_trials: bool, trials_with_values: Optional[Sequence[Tuple[FrozenTrial, Sequence[float]]]], hovertemplate: str, infeasible: bool = False, dominated_trials: bool = False, ) -> Union["go.Scatter", "go.Scatter3d"]: trials_with_values = trials_with_values or [] assert n_targets in (2, 3) marker = _make_marker( [trial for trial, _ in trials_with_values], include_dominated_trials, dominated_trials=dominated_trials, infeasible=infeasible, ) if n_targets == 2: return go.Scatter( x=[values[axis_order[0]] for _, values in trials_with_values], y=[values[axis_order[1]] for _, values in trials_with_values], text=[_make_hovertext(trial) for trial, _ in trials_with_values], mode="markers", hovertemplate=hovertemplate, marker=marker, showlegend=False, ) else: assert n_targets == 3 return go.Scatter3d( x=[values[axis_order[0]] for _, values in trials_with_values], y=[values[axis_order[1]] for _, values in trials_with_values], z=[values[axis_order[2]] for _, values in trials_with_values], text=[_make_hovertext(trial) for trial, _ in trials_with_values], mode="markers", hovertemplate=hovertemplate, marker=marker, showlegend=False, )
def _generate_slice_subplot(subplot_info: _SliceSubplotInfo) -> "Scatter": return go.Scatter( x=subplot_info.x, y=subplot_info.y, mode="markers", marker={ "line": { "width": 0.5, "color": "Grey" }, "color": subplot_info.trial_numbers, "colorscale": COLOR_SCALE, "colorbar": { "title": "Trial", "x": 1.0, # Offset the colorbar position with a fixed width `xpad`. "xpad": 40, }, }, showlegend=False, )
def _get_intermediate_plot(study: Study) -> "go.Figure": layout = go.Layout( title="Intermediate Values Plot", xaxis={"title": "Step"}, yaxis={"title": "Intermediate Value"}, showlegend=False, ) target_state = [TrialState.PRUNED, TrialState.COMPLETE, TrialState.RUNNING] trials = [trial for trial in study.trials if trial.state in target_state] if len(trials) == 0: _logger.warning("Study instance does not contain trials.") return go.Figure(data=[], layout=layout) traces = [] for trial in trials: if trial.intermediate_values: sorted_intermediate_values = sorted( trial.intermediate_values.items()) trace = go.Scatter( x=tuple((x for x, _ in sorted_intermediate_values)), y=tuple((y for _, y in sorted_intermediate_values)), mode="lines+markers", marker={"maxdisplayed": 10}, name="Trial{}".format(trial.number), ) traces.append(trace) if not traces: _logger.warning( "You need to set up the pruning feature to utilize `plot_intermediate_values()`" ) return go.Figure(data=[], layout=layout) figure = go.Figure(data=traces, layout=layout) return figure
def _make_scatter_object_base( n_dim: int, trials: Sequence[FrozenTrial], axis_order: List[int], include_dominated_trials: bool, hovertemplate: str, infeasible: bool = False, dominated_trials: bool = False, ) -> Union["go.Scatter", "go.Scatter3d"]: assert n_dim in (2, 3) marker = _make_marker( trials, include_dominated_trials, dominated_trials=dominated_trials, infeasible=infeasible, ) if n_dim == 2: return go.Scatter( x=[t.values[axis_order[0]] for t in trials], y=[t.values[axis_order[1]] for t in trials], text=[_make_hovertext(t) for t in trials], mode="markers", hovertemplate=hovertemplate, marker=marker, showlegend=False, ) else: assert n_dim == 3 return go.Scatter3d( x=[t.values[axis_order[0]] for t in trials], y=[t.values[axis_order[1]] for t in trials], z=[t.values[axis_order[2]] for t in trials], text=[_make_hovertext(t) for t in trials], mode="markers", hovertemplate=hovertemplate, marker=marker, showlegend=False, )
def _get_contour_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout(title="Contour Plot") trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(all_params) elif len(params) <= 1: _logger.warning("The length of params must be greater than 1.") return go.Figure(data=[], layout=layout) else: for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) sorted_params = sorted(set(params)) padding_ratio = 0.05 param_values_range = {} for p_name in sorted_params: values = _get_param_values(trials, p_name) min_value = min(values) max_value = max(values) if _is_log_scale(trials, p_name): padding = (math.log10(max_value) - math.log10(min_value)) * padding_ratio min_value = math.pow(10, math.log10(min_value) - padding) max_value = math.pow(10, math.log10(max_value) + padding) elif _is_numerical(trials, p_name): padding = (max_value - min_value) * padding_ratio min_value = min_value - padding max_value = max_value + padding else: # Plotly>=4.12.0 draws contours using the indices of categorical variables instead of # raw values and the range should be updated based on the cardinality of categorical # variables. See https://github.com/optuna/optuna/issues/1967. if version.parse(plotly.__version__) >= version.parse("4.12.0"): span = len(set(values)) - 1 padding = span * padding_ratio min_value = -padding max_value = span + padding param_values_range[p_name] = (min_value, max_value) reverse_scale = _is_reverse_scale(study, target) if len(sorted_params) == 2: x_param = sorted_params[0] y_param = sorted_params[1] sub_plots = _generate_contour_subplot(trials, x_param, y_param, reverse_scale, param_values_range, target, target_name) figure = go.Figure(data=sub_plots, layout=layout) figure.update_xaxes(title_text=x_param, range=param_values_range[x_param]) figure.update_yaxes(title_text=y_param, range=param_values_range[y_param]) if not _is_numerical(trials, x_param): figure.update_xaxes(type="category") if not _is_numerical(trials, y_param): figure.update_yaxes(type="category") if _is_log_scale(trials, x_param): log_range = [math.log10(p) for p in param_values_range[x_param]] figure.update_xaxes(range=log_range, type="log") if _is_log_scale(trials, y_param): log_range = [math.log10(p) for p in param_values_range[y_param]] figure.update_yaxes(range=log_range, type="log") else: figure = make_subplots(rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True) figure.update_layout(layout) showscale = True # showscale option only needs to be specified once for x_i, x_param in enumerate(sorted_params): for y_i, y_param in enumerate(sorted_params): if x_param == y_param: figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1) else: sub_plots = _generate_contour_subplot( trials, x_param, y_param, reverse_scale, param_values_range, target, target_name, ) contour = sub_plots[0] scatter = sub_plots[1] contour.update( showscale=showscale) # showscale's default is True if showscale: showscale = False figure.add_trace(contour, row=y_i + 1, col=x_i + 1) figure.add_trace(scatter, row=y_i + 1, col=x_i + 1) figure.update_xaxes(range=param_values_range[x_param], row=y_i + 1, col=x_i + 1) figure.update_yaxes(range=param_values_range[y_param], row=y_i + 1, col=x_i + 1) if not _is_numerical(trials, x_param): figure.update_xaxes(type="category", row=y_i + 1, col=x_i + 1) if not _is_numerical(trials, y_param): figure.update_yaxes(type="category", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, x_param): log_range = [ math.log10(p) for p in param_values_range[x_param] ] figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, y_param): log_range = [ math.log10(p) for p in param_values_range[y_param] ] figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if x_i == 0: figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1) if y_i == len(sorted_params) - 1: figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1) return figure
def _generate_contour_subplot( trials: List[FrozenTrial], x_param: str, y_param: str, reverse_scale: bool, param_values_range: Optional[Dict[str, Tuple[float, float]]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> Tuple["Contour", "Scatter"]: if param_values_range is None: param_values_range = {} x_indices = sorted(set(_get_param_values(trials, x_param))) y_indices = sorted(set(_get_param_values(trials, y_param))) if len(x_indices) < 2: _logger.warning( "Param {} unique value length is less than 2.".format(x_param)) return go.Contour(), go.Scatter() if len(y_indices) < 2: _logger.warning( "Param {} unique value length is less than 2.".format(y_param)) return go.Contour(), go.Scatter() # Padding to the plot for non-categorical params. x_range = param_values_range[x_param] if _is_numerical(trials, x_param): x_indices = [x_range[0]] + x_indices + [x_range[1]] y_range = param_values_range[y_param] if _is_numerical(trials, y_param): y_indices = [y_range[0]] + y_indices + [y_range[1]] z = [[float("nan") for _ in range(len(x_indices))] for _ in range(len(y_indices))] x_values = [] y_values = [] for trial in trials: if x_param not in trial.params or y_param not in trial.params: continue x_value = trial.params[x_param] y_value = trial.params[y_param] if not _is_numerical(trials, x_param): x_value = str(x_value) if not _is_numerical(trials, y_param): y_value = str(y_value) x_values.append(x_value) y_values.append(y_value) x_i = x_indices.index(x_value) y_i = y_indices.index(y_value) if target is None: value = trial.value else: value = target(trial) if isinstance(value, int): value = float(value) elif not isinstance(value, float): raise ValueError( f"Trial{trial.number} has COMPLETE state, but its target value is non-numeric." ) z[y_i][x_i] = value contour = go.Contour( x=x_indices, y=y_indices, z=z, colorbar={"title": target_name}, colorscale=COLOR_SCALE, connectgaps=True, contours_coloring="heatmap", hoverinfo="none", line_smoothing=1.3, reversescale=reverse_scale, ) scatter = go.Scatter( x=x_values, y=y_values, marker={ "line": { "width": 2.0, "color": "Grey" }, "color": "black" }, mode="markers", showlegend=False, ) return (contour, scatter)
def _get_contour_plot(study: Study, params: Optional[List[str]] = None) -> "go.Figure": layout = go.Layout(title="Contour Plot") trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE] if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(list(all_params)) elif len(params) <= 1: _logger.warning("The length of params must be greater than 1.") return go.Figure(data=[], layout=layout) else: for input_p_name in params: if input_p_name not in all_params: raise ValueError("Parameter {} does not exist in your study.".format(input_p_name)) sorted_params = sorted(list(set(params))) padding_ratio = 0.05 param_values_range = {} update_category_axes = {} for p_name in sorted_params: values = _get_param_values(trials, p_name) min_value = min(values) max_value = max(values) if _is_log_scale(trials, p_name): padding = (math.log10(max_value) - math.log10(min_value)) * padding_ratio min_value = math.pow(10, math.log10(min_value) - padding) max_value = math.pow(10, math.log10(max_value) + padding) elif _is_categorical(trials, p_name): # For numeric values, plotly does not automatically plot as "category" type. update_category_axes[p_name] = any([str(v).isnumeric() for v in set(values)]) else: padding = (max_value - min_value) * padding_ratio min_value = min_value - padding max_value = max_value + padding param_values_range[p_name] = (min_value, max_value) if len(sorted_params) == 2: x_param = sorted_params[0] y_param = sorted_params[1] sub_plots = _generate_contour_subplot( trials, x_param, y_param, study.direction, param_values_range ) figure = go.Figure(data=sub_plots, layout=layout) figure.update_xaxes(title_text=x_param, range=param_values_range[x_param]) figure.update_yaxes(title_text=y_param, range=param_values_range[y_param]) if update_category_axes.get(x_param, False): figure.update_xaxes(type="category") if update_category_axes.get(y_param, False): figure.update_yaxes(type="category") if _is_log_scale(trials, x_param): log_range = [math.log10(p) for p in param_values_range[x_param]] figure.update_xaxes(range=log_range, type="log") if _is_log_scale(trials, y_param): log_range = [math.log10(p) for p in param_values_range[y_param]] figure.update_yaxes(range=log_range, type="log") else: figure = make_subplots( rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True ) figure.update_layout(layout) showscale = True # showscale option only needs to be specified once for x_i, x_param in enumerate(sorted_params): for y_i, y_param in enumerate(sorted_params): if x_param == y_param: figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1) else: sub_plots = _generate_contour_subplot( trials, x_param, y_param, study.direction, param_values_range ) contour = sub_plots[0] scatter = sub_plots[1] contour.update(showscale=showscale) # showscale's default is True if showscale: showscale = False figure.add_trace(contour, row=y_i + 1, col=x_i + 1) figure.add_trace(scatter, row=y_i + 1, col=x_i + 1) figure.update_xaxes(range=param_values_range[x_param], row=y_i + 1, col=x_i + 1) figure.update_yaxes(range=param_values_range[y_param], row=y_i + 1, col=x_i + 1) if update_category_axes.get(x_param, False): figure.update_xaxes(type="category", row=y_i + 1, col=x_i + 1) if update_category_axes.get(y_param, False): figure.update_yaxes(type="category", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, x_param): log_range = [math.log10(p) for p in param_values_range[x_param]] figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, y_param): log_range = [math.log10(p) for p in param_values_range[y_param]] figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if x_i == 0: figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1) if y_i == len(sorted_params) - 1: figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1) return figure
def _generate_contour_subplot( trials: List[FrozenTrial], x_param: str, y_param: str, direction: StudyDirection, param_values_range: Optional[Dict[str, Tuple[float, float]]] = None, ) -> Tuple["Contour", "Scatter"]: if param_values_range is None: param_values_range = {} x_indices = sorted(set(_get_param_values(trials, x_param))) y_indices = sorted(set(_get_param_values(trials, y_param))) if len(x_indices) < 2: _logger.warning("Param {} unique value length is less than 2.".format(x_param)) return go.Contour(), go.Scatter() if len(y_indices) < 2: _logger.warning("Param {} unique value length is less than 2.".format(y_param)) return go.Contour(), go.Scatter() # Padding to the plot for non-categorical params. x_range = param_values_range[x_param] if not _is_categorical(trials, x_param): x_indices = [x_range[0]] + x_indices + [x_range[1]] y_range = param_values_range[y_param] if not _is_categorical(trials, y_param): y_indices = [y_range[0]] + y_indices + [y_range[1]] z = [[float("nan") for _ in range(len(x_indices))] for _ in range(len(y_indices))] x_values = [] y_values = [] for trial in trials: if x_param not in trial.params or y_param not in trial.params: continue x_value = trial.params[x_param] y_value = trial.params[y_param] if _is_categorical(trials, x_param): x_value = str(x_value) if _is_categorical(trials, y_param): y_value = str(y_value) x_values.append(x_value) y_values.append(y_value) x_i = x_indices.index(x_value) y_i = y_indices.index(y_value) if isinstance(trial.value, int): value = float(trial.value) elif isinstance(trial.value, float): value = trial.value else: raise ValueError( "Trial{} has COMPLETE state, but its value is non-numeric.".format(trial.number) ) z[y_i][x_i] = value # TODO(Yanase): Use reversescale argument to reverse colorscale if Plotly's bug is fixed. # If contours_coloring='heatmap' is specified, reversescale argument of go.Contour does not # work correctly. See https://github.com/pfnet/optuna/issues/606. colorscale = plotly.colors.PLOTLY_SCALES["Blues"] if direction == StudyDirection.MAXIMIZE: colorscale = [[1 - t[0], t[1]] for t in colorscale] colorscale.reverse() contour = go.Contour( x=x_indices, y=y_indices, z=z, colorbar={"title": "Objective Value"}, colorscale=colorscale, connectgaps=True, contours_coloring="heatmap", hoverinfo="none", line_smoothing=1.3, ) scatter = go.Scatter( x=x_values, y=y_values, marker={"line": {"width": 0.5, "color": "Grey"}, "color": "black"}, mode="markers", showlegend=False, ) return (contour, scatter)
def _get_pareto_front_2d( study: Study, target_names: Optional[List[str]], include_dominated_trials: bool = False, axis_order: Optional[List[int]] = None, ) -> "go.Figure": if target_names is None: target_names = ["Objective 0", "Objective 1"] elif len(target_names) != 2: raise ValueError("The length of `target_names` is supposed to be 2.") trials = study.best_trials n_best_trials = len(trials) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") if include_dominated_trials: non_pareto_trials = _get_non_pareto_front_trials(study, trials) trials += non_pareto_trials if axis_order is None: axis_order = list(range(2)) else: if len(axis_order) != 2: raise ValueError( f"Size of `axis_order` {axis_order}. Expect: 2, Actual: {len(axis_order)}." ) if len(set(axis_order)) != 2: raise ValueError(f"Elements of given `axis_order` {axis_order} are not unique!") if max(axis_order) > 1: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} " "higher than 1." ) if min(axis_order) < 0: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} " "lower than 0." ) data = [ go.Scatter( x=[t.values[axis_order[0]] for t in trials[n_best_trials:]], y=[t.values[axis_order[1]] for t in trials[n_best_trials:]], text=[_make_hovertext(t) for t in trials[n_best_trials:]], mode="markers", hovertemplate="%{text}<extra>Trial</extra>", marker={ "line": {"width": 0.5, "color": "Grey"}, "color": [t.number for t in trials[n_best_trials:]], "colorscale": "Blues", "colorbar": { "title": "#Trials", }, }, showlegend=False, ), go.Scatter( x=[t.values[axis_order[0]] for t in trials[:n_best_trials]], y=[t.values[axis_order[1]] for t in trials[:n_best_trials]], text=[_make_hovertext(t) for t in trials[:n_best_trials]], mode="markers", hovertemplate="%{text}<extra>Best Trial</extra>", marker={ "line": {"width": 0.5, "color": "Grey"}, "color": [t.number for t in trials[:n_best_trials]], "colorscale": "Reds", "colorbar": { "title": "#Best trials", "x": 1.1 if include_dominated_trials else 1, "xpad": 40, }, }, showlegend=False, ), ] layout = go.Layout( title="Pareto-front Plot", xaxis_title=target_names[axis_order[0]], yaxis_title=target_names[axis_order[1]], ) return go.Figure(data=data, layout=layout)
def plot_edf( study: Union[Study, Sequence[Study]], *, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": """Plot the objective value EDF (empirical distribution function) of a study. Note that only the complete trials are considered when plotting the EDF. .. note:: EDF is useful to analyze and improve search spaces. For instance, you can see a practical use case of EDF in the paper `Designing Network Design Spaces <https://arxiv.org/abs/2003.13678>`_. .. note:: The plotted EDF assumes that the value of the objective function is in accordance with the uniform distribution over the objective space. Example: The following code snippet shows how to plot EDF. .. plotly:: import math import optuna def ackley(x, y): a = 20 * math.exp(-0.2 * math.sqrt(0.5 * (x ** 2 + y ** 2))) b = math.exp(0.5 * (math.cos(2 * math.pi * x) + math.cos(2 * math.pi * y))) return -a - b + math.e + 20 def objective(trial, low, high): x = trial.suggest_float("x", low, high) y = trial.suggest_float("y", low, high) return ackley(x, y) sampler = optuna.samplers.RandomSampler(seed=10) # Widest search space. study0 = optuna.create_study(study_name="x=[0,5), y=[0,5)", sampler=sampler) study0.optimize(lambda t: objective(t, 0, 5), n_trials=500) # Narrower search space. study1 = optuna.create_study(study_name="x=[0,4), y=[0,4)", sampler=sampler) study1.optimize(lambda t: objective(t, 0, 4), n_trials=500) # Narrowest search space but it doesn't include the global optimum point. study2 = optuna.create_study(study_name="x=[1,3), y=[1,3)", sampler=sampler) study2.optimize(lambda t: objective(t, 1, 3), n_trials=500) fig = optuna.visualization.plot_edf([study0, study1, study2]) fig.show() Args: study: A target :class:`~optuna.study.Study` object. You can pass multiple studies if you want to compare those EDFs. target: A function to specify the value to display. If it is :obj:`None` and ``study`` is being used for single-objective optimization, the objective values are plotted. .. note:: Specify this argument if ``study`` is being used for multi-objective optimization. target_name: Target's name to display on the axis label. Returns: A :class:`plotly.graph_objs.Figure` object. """ _imports.check() layout = go.Layout( title="Empirical Distribution Function Plot", xaxis={"title": target_name}, yaxis={"title": "Cumulative Probability"}, ) info = _get_edf_info(study, target, target_name) edf_lines = info.lines if len(edf_lines) == 0: return go.Figure(data=[], layout=layout) traces = [] for study_name, y_values in edf_lines: traces.append( go.Scatter(x=info.x_values, y=y_values, name=study_name, mode="lines")) figure = go.Figure(data=traces, layout=layout) figure.update_yaxes(range=[0, 1]) return figure
def _get_optimization_histories_with_error_bar( studies: List[Study], target: Optional[Callable[[FrozenTrial], float]], target_name: str, layout: "go.Layout", ) -> "go.Figure": max_trial_number = np.max( [ trial.number for study in studies for trial in study.get_trials(states=(TrialState.COMPLETE,)) ] ) _target: Callable[[FrozenTrial], float] if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) else: _target = target target_values: List[List[float]] = [[] for _ in range(max_trial_number + 2)] for study in studies: trials = study.get_trials(states=(TrialState.COMPLETE,)) for t in trials: target_values[t.number].append(_target(t)) mean_of_target_values = [np.mean(v) if len(v) > 0 else None for v in target_values] std_of_target_values = [np.std(v) if len(v) > 0 else None for v in target_values] trial_numbers = np.arange(max_trial_number + 2)[[v is not None for v in mean_of_target_values]] means = np.asarray(mean_of_target_values)[trial_numbers] stds = np.asarray(std_of_target_values)[trial_numbers] traces = [ go.Scatter( x=trial_numbers, y=means, error_y={ "type": "data", "array": stds, "visible": True, }, mode="markers", name=target_name, ) ] if target is None: best_values: List[List[float]] = [[] for _ in range(max_trial_number + 2)] for study in studies: trials = study.get_trials(states=(TrialState.COMPLETE,)) if study.direction == StudyDirection.MINIMIZE: best_vs = np.minimum.accumulate([cast(float, t.value) for t in trials]) else: best_vs = np.maximum.accumulate([cast(float, t.value) for t in trials]) for i, t in enumerate(trials): best_values[t.number].append(best_vs[i]) mean_of_best_values = [np.mean(v) if len(v) > 0 else None for v in best_values] std_of_best_values = [np.std(v) if len(v) > 0 else None for v in best_values] means = np.asarray(mean_of_best_values)[trial_numbers] stds = np.asarray(std_of_best_values)[trial_numbers] traces.append(go.Scatter(x=trial_numbers, y=means, name="Best Value")) traces.append( go.Scatter( x=trial_numbers, y=means + stds, mode="lines", line=dict(width=0.01), showlegend=False, ) ) traces.append( go.Scatter( x=trial_numbers, y=means - stds, mode="none", showlegend=False, fill="tonexty", fillcolor="rgba(255,0,0,0.2)", ) ) figure = go.Figure(data=traces, layout=layout) return figure
def _get_optimization_history_plot( info_list: List[_OptimizationHistoryInfo], target_name: str, ) -> "go.Figure": layout = go.Layout( title="Optimization History Plot", xaxis={"title": "Trial"}, yaxis={"title": target_name}, ) traces = [] for trial_numbers, values_info, best_values_info in info_list: if values_info.stds is None: error_y = None else: error_y = { "type": "data", "array": values_info.stds, "visible": True } traces.append( go.Scatter( x=trial_numbers, y=values_info.values, error_y=error_y, mode="markers", name=values_info.label_name, )) if best_values_info is not None: traces.append( go.Scatter( x=trial_numbers, y=best_values_info.values, name=best_values_info.label_name, )) if best_values_info.stds is not None: upper = np.array(best_values_info.values) + np.array( best_values_info.stds) traces.append( go.Scatter( x=trial_numbers, y=upper, mode="lines", line=dict(width=0.01), showlegend=False, )) lower = np.array(best_values_info.values) - np.array( best_values_info.stds) traces.append( go.Scatter( x=trial_numbers, y=lower, mode="none", showlegend=False, fill="tonexty", fillcolor="rgba(255,0,0,0.2)", )) return go.Figure(data=traces, layout=layout)
def _get_contour_plot(info: _ContourInfo) -> "go.Figure": layout = go.Layout(title="Contour Plot") sorted_params = info.sorted_params sub_plot_infos = info.sub_plot_infos reverse_scale = info.reverse_scale target_name = info.target_name if len(sorted_params) <= 1: return go.Figure(data=[], layout=layout) if len(sorted_params) == 2: x_param = sorted_params[0] y_param = sorted_params[1] sub_plot_info = sub_plot_infos[0][0] sub_plots = _get_contour_subplot(sub_plot_info, reverse_scale, target_name) figure = go.Figure(data=sub_plots, layout=layout) figure.update_xaxes(title_text=x_param, range=sub_plot_info.xaxis.range) figure.update_yaxes(title_text=y_param, range=sub_plot_info.yaxis.range) if sub_plot_info.xaxis.is_cat: figure.update_xaxes(type="category") if sub_plot_info.yaxis.is_cat: figure.update_yaxes(type="category") if sub_plot_info.xaxis.is_log: log_range = [math.log10(p) for p in sub_plot_info.xaxis.range] figure.update_xaxes(range=log_range, type="log") if sub_plot_info.yaxis.is_log: log_range = [math.log10(p) for p in sub_plot_info.yaxis.range] figure.update_yaxes(range=log_range, type="log") else: figure = make_subplots( rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True ) figure.update_layout(layout) showscale = True # showscale option only needs to be specified once. for x_i, x_param in enumerate(sorted_params): for y_i, y_param in enumerate(sorted_params): if x_param == y_param: figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1) else: sub_plots = _get_contour_subplot( sub_plot_infos[y_i][x_i], reverse_scale, target_name ) contour = sub_plots[0] scatter = sub_plots[1] contour.update(showscale=showscale) # showscale's default is True. if showscale: showscale = False figure.add_trace(contour, row=y_i + 1, col=x_i + 1) figure.add_trace(scatter, row=y_i + 1, col=x_i + 1) xaxis = sub_plot_infos[y_i][x_i].xaxis yaxis = sub_plot_infos[y_i][x_i].yaxis figure.update_xaxes(range=xaxis.range, row=y_i + 1, col=x_i + 1) figure.update_yaxes(range=yaxis.range, row=y_i + 1, col=x_i + 1) if xaxis.is_cat: figure.update_xaxes(type="category", row=y_i + 1, col=x_i + 1) if yaxis.is_cat: figure.update_yaxes(type="category", row=y_i + 1, col=x_i + 1) if xaxis.is_log: log_range = [math.log10(p) for p in xaxis.range] figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if yaxis.is_log: log_range = [math.log10(p) for p in yaxis.range] figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if x_i == 0: figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1) if y_i == len(sorted_params) - 1: figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1) return figure
def _generate_contour_subplot( trials: List[FrozenTrial], x_param: str, y_param: str, direction: StudyDirection) -> Tuple["Contour", "Scatter"]: x_indices = sorted( list({t.params[x_param] for t in trials if x_param in t.params})) y_indices = sorted( list({t.params[y_param] for t in trials if y_param in t.params})) if len(x_indices) < 2: _logger.warning( "Param {} unique value length is less than 2.".format(x_param)) return go.Contour(), go.Scatter() if len(y_indices) < 2: _logger.warning( "Param {} unique value length is less than 2.".format(y_param)) return go.Contour(), go.Scatter() z = [[float("nan") for _ in range(len(x_indices))] for _ in range(len(y_indices))] x_values = [] y_values = [] for trial in trials: if x_param not in trial.params or y_param not in trial.params: continue x_values.append(trial.params[x_param]) y_values.append(trial.params[y_param]) x_i = x_indices.index(trial.params[x_param]) y_i = y_indices.index(trial.params[y_param]) if isinstance(trial.value, int): value = float(trial.value) elif isinstance(trial.value, float): value = trial.value else: raise ValueError( "Trial{} has COMPLETE state, but its value is non-numeric.". format(trial.number)) z[y_i][x_i] = value # TODO(Yanase): Use reversescale argument to reverse colorscale if Plotly's bug is fixed. # If contours_coloring='heatmap' is specified, reversesecale argument of go.Contour does not # work correctly. See https://github.com/pfnet/optuna/issues/606. colorscale = plotly.colors.PLOTLY_SCALES["Blues"] if direction == StudyDirection.MINIMIZE: colorscale = [[1 - t[0], t[1]] for t in colorscale] colorscale.reverse() contour = go.Contour( x=x_indices, y=y_indices, z=z, colorbar={"title": "Objective Value"}, colorscale=colorscale, connectgaps=True, contours_coloring="heatmap", hoverinfo="none", line_smoothing=1.3, ) scatter = go.Scatter(x=x_values, y=y_values, marker={"color": "black"}, mode="markers", showlegend=False) return (contour, scatter)