def plot_param_importances( study: Study, evaluator: Optional[BaseImportanceEvaluator] = None, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": """Plot hyperparameter importances. Example: The following code snippet shows how to plot hyperparameter importances. .. plotly:: import optuna def objective(trial): x = trial.suggest_int("x", 0, 2) y = trial.suggest_float("y", -1.0, 1.0) z = trial.suggest_float("z", 0.0, 1.5) return x ** 2 + y ** 3 - z ** 4 sampler = optuna.samplers.RandomSampler(seed=10) study = optuna.create_study(sampler=sampler) study.optimize(objective, n_trials=100) fig = optuna.visualization.plot_param_importances(study) fig.show() .. seealso:: This function visualizes the results of :func:`optuna.importance.get_param_importances`. Args: study: An optimized study. evaluator: An importance evaluator object that specifies which algorithm to base the importance assessment on. Defaults to :class:`~optuna.importance.FanovaImportanceEvaluator`. params: A list of names of parameters to assess. If :obj:`None`, all parameters that are present in all of the completed trials are assessed. target: A function to specify the value to display. If it is :obj:`None` and ``study`` is being used for single-objective optimization, the objective values are plotted. .. note:: Specify this argument if ``study`` is being used for multi-objective optimization. target_name: Target's name to display on the axis label. Returns: A :class:`plotly.graph_objs.Figure` object. Raises: :exc:`ValueError`: If ``target`` is :obj:`None` and ``study`` is being used for multi-objective optimization. """ _imports.check() _check_plot_args(study, target, target_name) layout = go.Layout( title="Hyperparameter Importances", xaxis={"title": f"Importance for {target_name}"}, yaxis={"title": "Hyperparameter"}, showlegend=False, ) # Importances cannot be evaluated without completed trials. # Return an empty figure for consistency with other visualization functions. trials = [ trial for trial in study.trials if trial.state == TrialState.COMPLETE ] if len(trials) == 0: logger.warning("Study instance does not contain completed trials.") return go.Figure(data=[], layout=layout) importances = optuna.importance.get_param_importances(study, evaluator=evaluator, params=params, target=target) importances = OrderedDict(reversed(list(importances.items()))) importance_values = list(importances.values()) param_names = list(importances.keys()) fig = go.Figure( data=[ go.Bar( x=importance_values, y=param_names, text=importance_values, texttemplate="%{text:.2f}", textposition="outside", cliponaxis=False, # Ensure text is not clipped. hovertemplate=[ _make_hovertext(param_name, importance, study) for param_name, importance in importances.items() ], marker_color=[ _get_color(param_name, study) for param_name in param_names ], orientation="h", ) ], layout=layout, ) return fig
def _get_pareto_front_3d( study: Study, target_names: Optional[List[str]], include_dominated_trials: bool = False, axis_order: Optional[List[int]] = None, ) -> "go.Figure": if target_names is None: target_names = ["Objective 0", "Objective 1", "Objective 2"] elif len(target_names) != 3: raise ValueError("The length of `target_names` is supposed to be 3.") trials = study.best_trials n_best_trials = len(trials) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") if include_dominated_trials: non_pareto_trials = _get_non_pareto_front_trials(study, trials) trials += non_pareto_trials if axis_order is None: axis_order = list(range(3)) else: if len(axis_order) != 3: raise ValueError( f"Size of `axis_order` {axis_order}. Expect: 3, Actual: {len(axis_order)}." ) if len(set(axis_order)) != 3: raise ValueError(f"Elements of given `axis_order` {axis_order} are not unique!.") if max(axis_order) > 2: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} " "higher than 2." ) if min(axis_order) < 0: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} " "lower than 0." ) data = [ go.Scatter3d( x=[t.values[axis_order[0]] for t in trials[n_best_trials:]], y=[t.values[axis_order[1]] for t in trials[n_best_trials:]], z=[t.values[axis_order[2]] for t in trials[n_best_trials:]], text=[_make_hovertext(t) for t in trials[n_best_trials:]], hovertemplate="%{text}<extra>Trial</extra>", mode="markers", marker={ "line": {"width": 0.5, "color": "Grey"}, "color": [t.number for t in trials[n_best_trials:]], "colorscale": "Blues", "colorbar": { "title": "#Trials", }, }, showlegend=False, ), go.Scatter3d( x=[t.values[axis_order[0]] for t in trials[:n_best_trials]], y=[t.values[axis_order[1]] for t in trials[:n_best_trials]], z=[t.values[axis_order[2]] for t in trials[:n_best_trials]], text=[_make_hovertext(t) for t in trials[:n_best_trials]], hovertemplate="%{text}<extra>Best Trial</extra>", mode="markers", marker={ "line": {"width": 0.5, "color": "Grey"}, "color": [t.number for t in trials[:n_best_trials]], "colorscale": "Reds", "colorbar": { "title": "#Best trials", "x": 1.1 if include_dominated_trials else 1, "xpad": 40, }, }, showlegend=False, ), ] layout = go.Layout( title="Pareto-front Plot", scene={ "xaxis_title": target_names[axis_order[0]], "yaxis_title": target_names[axis_order[1]], "zaxis_title": target_names[axis_order[2]], }, ) return go.Figure(data=data, layout=layout)
def _get_contour_plot(study: Study, params: Optional[List[str]] = None) -> "go.Figure": layout = go.Layout(title="Contour Plot") trials = [ trial for trial in study.trials if trial.state == TrialState.COMPLETE ] if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(list(all_params)) elif len(params) <= 1: _logger.warning("The length of params must be greater than 1.") return go.Figure(data=[], layout=layout) else: for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) sorted_params = sorted(list(set(params))) padding_ratio = 0.05 param_values_range = {} update_category_axes = {} for p_name in sorted_params: values = _get_param_values(trials, p_name) min_value = min(values) max_value = max(values) if _is_log_scale(trials, p_name): padding = (math.log10(max_value) - math.log10(min_value)) * padding_ratio min_value = math.pow(10, math.log10(min_value) - padding) max_value = math.pow(10, math.log10(max_value) + padding) elif _is_categorical(trials, p_name): # For numeric values, plotly does not automatically plot as "category" type. update_category_axes[p_name] = any( [str(v).isnumeric() for v in set(values)]) # Plotly>=4.12.0 draws contours using the indices of categorical variables instead of # raw values and the range should be updated based on the cardinality of categorical # variables. See https://github.com/optuna/optuna/issues/1967. if version.parse(plotly.__version__) >= version.parse("4.12.0"): span = len(set(values)) - 1 padding = span * padding_ratio min_value = -padding max_value = span + padding else: padding = (max_value - min_value) * padding_ratio min_value = min_value - padding max_value = max_value + padding param_values_range[p_name] = (min_value, max_value) if len(sorted_params) == 2: x_param = sorted_params[0] y_param = sorted_params[1] sub_plots = _generate_contour_subplot(trials, x_param, y_param, study.direction, param_values_range) figure = go.Figure(data=sub_plots, layout=layout) figure.update_xaxes(title_text=x_param, range=param_values_range[x_param]) figure.update_yaxes(title_text=y_param, range=param_values_range[y_param]) if update_category_axes.get(x_param, False): figure.update_xaxes(type="category") if update_category_axes.get(y_param, False): figure.update_yaxes(type="category") if _is_log_scale(trials, x_param): log_range = [math.log10(p) for p in param_values_range[x_param]] figure.update_xaxes(range=log_range, type="log") if _is_log_scale(trials, y_param): log_range = [math.log10(p) for p in param_values_range[y_param]] figure.update_yaxes(range=log_range, type="log") else: figure = make_subplots(rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True) figure.update_layout(layout) showscale = True # showscale option only needs to be specified once for x_i, x_param in enumerate(sorted_params): for y_i, y_param in enumerate(sorted_params): if x_param == y_param: figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1) else: sub_plots = _generate_contour_subplot( trials, x_param, y_param, study.direction, param_values_range) contour = sub_plots[0] scatter = sub_plots[1] contour.update( showscale=showscale) # showscale's default is True if showscale: showscale = False figure.add_trace(contour, row=y_i + 1, col=x_i + 1) figure.add_trace(scatter, row=y_i + 1, col=x_i + 1) figure.update_xaxes(range=param_values_range[x_param], row=y_i + 1, col=x_i + 1) figure.update_yaxes(range=param_values_range[y_param], row=y_i + 1, col=x_i + 1) if update_category_axes.get(x_param, False): figure.update_xaxes(type="category", row=y_i + 1, col=x_i + 1) if update_category_axes.get(y_param, False): figure.update_yaxes(type="category", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, x_param): log_range = [ math.log10(p) for p in param_values_range[x_param] ] figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, y_param): log_range = [ math.log10(p) for p in param_values_range[y_param] ] figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if x_i == 0: figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1) if y_i == len(sorted_params) - 1: figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1) return figure
def _get_slice_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout(title="Slice Plot") trials = [ trial for trial in study.trials if trial.state == TrialState.COMPLETE ] if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(list(all_params)) else: for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) sorted_params = sorted(list(set(params))) n_params = len(sorted_params) if n_params == 1: figure = go.Figure(data=[ _generate_slice_subplot(study, trials, sorted_params[0], target) ], layout=layout) figure.update_xaxes(title_text=sorted_params[0]) figure.update_yaxes(title_text=target_name) if _is_log_scale(trials, sorted_params[0]): figure.update_xaxes(type="log") else: figure = make_subplots(rows=1, cols=len(sorted_params), shared_yaxes=True) figure.update_layout(layout) showscale = True # showscale option only needs to be specified once. for i, param in enumerate(sorted_params): trace = _generate_slice_subplot(study, trials, param, target) trace.update(marker={"showscale": showscale}) # showscale's default is True. if showscale: showscale = False figure.add_trace(trace, row=1, col=i + 1) figure.update_xaxes(title_text=param, row=1, col=i + 1) if i == 0: figure.update_yaxes(title_text=target_name, row=1, col=1) if _is_log_scale(trials, param): figure.update_xaxes(type="log", row=1, col=i + 1) if n_params > 3: # Ensure that each subplot has a minimum width without relying on autusizing. figure.update_layout(width=300 * n_params) return figure
def _get_parallel_coordinate_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout(title="Parallel Coordinate Plot") trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE] if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is not None: for input_p_name in params: if input_p_name not in all_params: raise ValueError("Parameter {} does not exist in your study.".format(input_p_name)) all_params = set(params) sorted_params = sorted(all_params) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target reversescale = study.direction == StudyDirection.MINIMIZE else: reversescale = True dims: List[Dict[str, Any]] = [ { "label": target_name, "values": tuple([target(t) for t in trials]), "range": (min([target(t) for t in trials]), max([target(t) for t in trials])), } ] numeric_cat_params_indices: List[int] = [] for dim_index, p_name in enumerate(sorted_params, start=1): values = [] for t in trials: if p_name in t.params: values.append(t.params[p_name]) if _is_log_scale(trials, p_name): values = [math.log10(v) for v in values] min_value = min(values) max_value = max(values) tickvals = list(range(math.ceil(min_value), math.ceil(max_value))) if min_value not in tickvals: tickvals = [min_value] + tickvals if max_value not in tickvals: tickvals = tickvals + [max_value] dim = { "label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]), "values": tuple(values), "range": (min_value, max_value), "tickvals": tickvals, "ticktext": ["{:.3g}".format(math.pow(10, x)) for x in tickvals], } elif _is_categorical(trials, p_name): vocab: DefaultDict[str, int] = defaultdict(lambda: len(vocab)) if _is_numerical(trials, p_name): _ = [vocab[v] for v in sorted(values)] values = [vocab[v] for v in values] ticktext = list(sorted(vocab.keys())) numeric_cat_params_indices.append(dim_index) else: values = [vocab[v] for v in values] ticktext = list(sorted(vocab.keys(), key=lambda x: vocab[x])) dim = { "label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]), "values": tuple(values), "range": (min(values), max(values)), "tickvals": list(range(len(vocab))), "ticktext": ticktext, } else: dim = { "label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]), "values": tuple(values), "range": (min(values), max(values)), } dims.append(dim) if numeric_cat_params_indices: # np.lexsort consumes the sort keys the order from back to front. # So the values of parameters have to be reversed the order. idx = np.lexsort([dims[index]["values"] for index in numeric_cat_params_indices][::-1]) for dim in dims: # Since the values are mapped to other categories by the index, # the index will be swapped according to the sorted index of numeric params. dim.update({"values": tuple(np.array(dim["values"])[idx])}) traces = [ go.Parcoords( dimensions=dims, labelangle=30, labelside="bottom", line={ "color": dims[0]["values"], "colorscale": "blues", "colorbar": {"title": target_name}, "showscale": True, "reversescale": reversescale, }, ) ] figure = go.Figure(data=traces, layout=layout) return figure
def plot_pareto_front( study: Study, *, target_names: Optional[List[str]] = None, include_dominated_trials: bool = True, axis_order: Optional[List[int]] = None, constraints_func: Optional[Callable[[FrozenTrial], Sequence[float]]] = None, targets: Optional[Callable[[FrozenTrial], Sequence[float]]] = None, ) -> "go.Figure": """Plot the Pareto front of a study. .. seealso:: Please refer to :ref:`multi_objective` for the tutorial of the Pareto front visualization. Example: The following code snippet shows how to plot the Pareto front of a study. .. plotly:: import optuna def objective(trial): x = trial.suggest_float("x", 0, 5) y = trial.suggest_float("y", 0, 3) v0 = 4 * x ** 2 + 4 * y ** 2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 study = optuna.create_study(directions=["minimize", "minimize"]) study.optimize(objective, n_trials=50) fig = optuna.visualization.plot_pareto_front(study) fig.show() Args: study: A :class:`~optuna.study.Study` object whose trials are plotted for their objective values. ``study.n_objectives`` must be either 2 or 3 when ``targets`` is :obj:`None`. target_names: Objective name list used as the axis titles. If :obj:`None` is specified, "Objective {objective_index}" is used instead. If ``targets`` is specified for a study that does not contain any completed trial, ``target_name`` must be specified. include_dominated_trials: A flag to include all dominated trial's objective values. axis_order: A list of indices indicating the axis order. If :obj:`None` is specified, default order is used. ``axis_order`` and ``targets`` cannot be used at the same time. constraints_func: An optional function that computes the objective constraints. It must take a :class:`~optuna.trial.FrozenTrial` and return the constraints. The return value must be a sequence of :obj:`float` s. A value strictly larger than 0 means that a constraint is violated. A value equal to or smaller than 0 is considered feasible. This specification is the same as in, for example, :class:`~optuna.integration.NSGAIISampler`. If given, trials are classified into three categories: feasible and best, feasible but non-best, and infeasible. Categories are shown in different colors. Here, whether a trial is best (on Pareto front) or not is determined ignoring all infeasible trials. targets: A function that returns targets values to display. The argument to this function is :class:`~optuna.trial.FrozenTrial`. ``axis_order`` and ``targets`` cannot be used at the same time. If ``study.n_objectives`` is neither 2 nor 3, ``targets`` must be specified. .. note:: Added in v3.0.0 as an experimental feature. The interface may change in newer versions without prior notice. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Returns: A :class:`plotly.graph_objs.Figure` object. """ _imports.check() info = _get_pareto_front_info(study, target_names, include_dominated_trials, axis_order, constraints_func, targets) if constraints_func is None: data = [ _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.non_best_trials_with_values, hovertemplate="%{text}<extra>Trial</extra>", dominated_trials=True, ), _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.best_trials_with_values, hovertemplate="%{text}<extra>Best Trial</extra>", dominated_trials=False, ), ] else: data = [ _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.infeasible_trials_with_values, hovertemplate="%{text}<extra>Infeasible Trial</extra>", infeasible=True, ), _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.non_best_trials_with_values, hovertemplate="%{text}<extra>Feasible Trial</extra>", dominated_trials=True, ), _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.best_trials_with_values, hovertemplate="%{text}<extra>Best Trial</extra>", dominated_trials=False, ), ] if info.n_targets == 2: layout = go.Layout( title="Pareto-front Plot", xaxis_title=info.target_names[info.axis_order[0]], yaxis_title=info.target_names[info.axis_order[1]], ) else: layout = go.Layout( title="Pareto-front Plot", scene={ "xaxis_title": info.target_names[info.axis_order[0]], "yaxis_title": info.target_names[info.axis_order[1]], "zaxis_title": info.target_names[info.axis_order[2]], }, ) return go.Figure(data=data, layout=layout)
def plot_edf( study: Union[Study, Sequence[Study]], *, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": """Plot the objective value EDF (empirical distribution function) of a study. Note that only the complete trials are considered when plotting the EDF. .. note:: EDF is useful to analyze and improve search spaces. For instance, you can see a practical use case of EDF in the paper `Designing Network Design Spaces <https://arxiv.org/abs/2003.13678>`_. .. note:: The plotted EDF assumes that the value of the objective function is in accordance with the uniform distribution over the objective space. Example: The following code snippet shows how to plot EDF. .. plotly:: import math import optuna def ackley(x, y): a = 20 * math.exp(-0.2 * math.sqrt(0.5 * (x ** 2 + y ** 2))) b = math.exp(0.5 * (math.cos(2 * math.pi * x) + math.cos(2 * math.pi * y))) return -a - b + math.e + 20 def objective(trial, low, high): x = trial.suggest_float("x", low, high) y = trial.suggest_float("y", low, high) return ackley(x, y) sampler = optuna.samplers.RandomSampler(seed=10) # Widest search space. study0 = optuna.create_study(study_name="x=[0,5), y=[0,5)", sampler=sampler) study0.optimize(lambda t: objective(t, 0, 5), n_trials=500) # Narrower search space. study1 = optuna.create_study(study_name="x=[0,4), y=[0,4)", sampler=sampler) study1.optimize(lambda t: objective(t, 0, 4), n_trials=500) # Narrowest search space but it doesn't include the global optimum point. study2 = optuna.create_study(study_name="x=[1,3), y=[1,3)", sampler=sampler) study2.optimize(lambda t: objective(t, 1, 3), n_trials=500) fig = optuna.visualization.plot_edf([study0, study1, study2]) fig.show() Args: study: A target :class:`~optuna.study.Study` object. You can pass multiple studies if you want to compare those EDFs. target: A function to specify the value to display. If it is :obj:`None` and ``study`` is being used for single-objective optimization, the objective values are plotted. .. note:: Specify this argument if ``study`` is being used for multi-objective optimization. target_name: Target's name to display on the axis label. Returns: A :class:`plotly.graph_objs.Figure` object. """ _imports.check() layout = go.Layout( title="Empirical Distribution Function Plot", xaxis={"title": target_name}, yaxis={"title": "Cumulative Probability"}, ) info = _get_edf_info(study, target, target_name) edf_lines = info.lines if len(edf_lines) == 0: return go.Figure(data=[], layout=layout) traces = [] for study_name, y_values in edf_lines: traces.append( go.Scatter(x=info.x_values, y=y_values, name=study_name, mode="lines")) figure = go.Figure(data=traces, layout=layout) figure.update_yaxes(range=[0, 1]) return figure
def _get_pareto_front_plot(info: _ParetoFrontInfo) -> "go.Figure": include_dominated_trials = info.include_dominated_trials has_constraints_func = info.has_constraints_func if not has_constraints_func: data = [ _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.non_best_trials_with_values, hovertemplate="%{text}<extra>Trial</extra>", dominated_trials=True, ), _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.best_trials_with_values, hovertemplate="%{text}<extra>Best Trial</extra>", dominated_trials=False, ), ] else: data = [ _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.infeasible_trials_with_values, hovertemplate="%{text}<extra>Infeasible Trial</extra>", infeasible=True, ), _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.non_best_trials_with_values, hovertemplate="%{text}<extra>Feasible Trial</extra>", dominated_trials=True, ), _make_scatter_object( info.n_targets, info.axis_order, include_dominated_trials, info.best_trials_with_values, hovertemplate="%{text}<extra>Best Trial</extra>", dominated_trials=False, ), ] if info.n_targets == 2: layout = go.Layout( title="Pareto-front Plot", xaxis_title=info.target_names[info.axis_order[0]], yaxis_title=info.target_names[info.axis_order[1]], ) else: layout = go.Layout( title="Pareto-front Plot", scene={ "xaxis_title": info.target_names[info.axis_order[0]], "yaxis_title": info.target_names[info.axis_order[1]], "zaxis_title": info.target_names[info.axis_order[2]], }, ) return go.Figure(data=data, layout=layout)
def _get_parallel_coordinate_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout(title="Parallel Coordinate Plot") trials = [ trial for trial in study.trials if trial.state == TrialState.COMPLETE ] if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is not None: for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) all_params = set(params) sorted_params = sorted(list(all_params)) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target reversescale = study.direction == StudyDirection.MINIMIZE else: reversescale = True dims: List[Dict[str, Any]] = [{ "label": target_name, "values": tuple([target(t) for t in trials]), "range": (min([target(t) for t in trials]), max([target(t) for t in trials])), }] for p_name in sorted_params: values = [] for t in trials: if p_name in t.params: values.append(t.params[p_name]) is_categorical = False try: tuple(map(float, values)) except (TypeError, ValueError): vocab: DefaultDict[str, int] = defaultdict(lambda: len(vocab)) values = [vocab[v] for v in values] is_categorical = True dim = { "label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]), "values": tuple(values), "range": (min(values), max(values)), } if is_categorical: dim["tickvals"] = list(range(len(vocab))) dim["ticktext"] = list(sorted(vocab.items(), key=lambda x: x[1])) dims.append(dim) traces = [ go.Parcoords( dimensions=dims, labelangle=30, labelside="bottom", line={ "color": dims[0]["values"], "colorscale": "blues", "colorbar": { "title": target_name }, "showscale": True, "reversescale": reversescale, }, ) ] figure = go.Figure(data=traces, layout=layout) return figure
def plot_param_importances( study: Study, evaluator: Optional[BaseImportanceEvaluator] = None, params: Optional[List[str]] = None, ) -> "go.Figure": """Plot hyperparameter importances. Example: The following code snippet shows how to plot hyperparameter importances. .. testcode:: import optuna def objective(trial): x = trial.suggest_int("x", 0, 2) y = trial.suggest_float("y", -1.0, 1.0) z = trial.suggest_float("z", 0.0, 1.5) return x ** 2 + y ** 3 - z ** 4 study = optuna.create_study(sampler=optuna.samplers.RandomSampler()) study.optimize(objective, n_trials=100) optuna.visualization.plot_param_importances(study) .. raw:: html <iframe src="../../_static/plot_param_importances.html" width="100%" height="500px" frameborder="0"> </iframe> .. seealso:: This function visualizes the results of :func:`optuna.importance.get_param_importances`. Args: study: An optimized study. evaluator: An importance evaluator object that specifies which algorithm to base the importance assessment on. Defaults to :class:`~optuna.importance.FanovaImportanceEvaluator`. params: A list of names of parameters to assess. If :obj:`None`, all parameters that are present in all of the completed trials are assessed. Returns: A :class:`plotly.graph_objs.Figure` object. """ _imports.check() layout = go.Layout( title="Hyperparameter Importances", xaxis={"title": "Importance"}, yaxis={"title": "Hyperparameter"}, showlegend=False, ) # Importances cannot be evaluated without completed trials. # Return an empty figure for consistency with other visualization functions. trials = [ trial for trial in study.trials if trial.state == TrialState.COMPLETE ] if len(trials) == 0: logger.warning("Study instance does not contain completed trials.") return go.Figure(data=[], layout=layout) importances = optuna.importance.get_param_importances(study, evaluator=evaluator, params=params) importances = OrderedDict(reversed(list(importances.items()))) importance_values = list(importances.values()) param_names = list(importances.keys()) fig = go.Figure( data=[ go.Bar( x=importance_values, y=param_names, text=importance_values, texttemplate="%{text:.2f}", textposition="outside", cliponaxis=False, # Ensure text is not clipped. hovertemplate=[ _make_hovertext(param_name, importance, study) for param_name, importance in importances.items() ], marker_color=[ _get_color(param_name, study) for param_name in param_names ], orientation="h", ) ], layout=layout, ) return fig
def _get_pareto_front_2d( study: Study, target_names: Optional[List[str]], include_dominated_trials: bool = False, axis_order: Optional[List[int]] = None, ) -> "go.Figure": if target_names is None: target_names = ["Objective 0", "Objective 1"] elif len(target_names) != 2: raise ValueError("The length of `target_names` is supposed to be 2.") trials = study.best_trials if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") if include_dominated_trials: non_pareto_trials = _get_non_pareto_front_trials(study, trials) trials += non_pareto_trials if axis_order is None: axis_order = list(range(2)) else: if len(axis_order) != 2: raise ValueError( f"Size of `axis_order` {axis_order}. Expect: 2, Actual: {len(axis_order)}." ) if len(set(axis_order)) != 2: raise ValueError(f"Elements of given `axis_order` {axis_order} are not unique!") if max(axis_order) > 1: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} " "higher than 1." ) if min(axis_order) < 0: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} " "lower than 0." ) data = [ go.Scatter( x=[t.values[axis_order[0]] for t in trials[len(study.best_trials) :]], y=[t.values[axis_order[1]] for t in trials[len(study.best_trials) :]], text=[_make_hovertext(t) for t in trials[len(study.best_trials) :]], mode="markers", hovertemplate="%{text}<extra>Trial</extra>", name="Trial", ), go.Scatter( x=[t.values[axis_order[0]] for t in trials[: len(study.best_trials)]], y=[t.values[axis_order[1]] for t in trials[: len(study.best_trials)]], text=[_make_hovertext(t) for t in trials[: len(study.best_trials)]], mode="markers", hovertemplate="%{text}<extra>Best Trial</extra>", name="Best Trial", ), ] layout = go.Layout( title="Pareto-front Plot", xaxis_title=target_names[axis_order[0]], yaxis_title=target_names[axis_order[1]], ) return go.Figure(data=data, layout=layout)
def _get_contour_plot(study: Study, params: Optional[List[str]] = None) -> "go.Figure": layout = go.Layout(title="Contour Plot") trials = [ trial for trial in study.trials if trial.state == TrialState.COMPLETE ] if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(list(all_params)) elif len(params) <= 1: _logger.warning("The length of params must be greater than 1.") return go.Figure(data=[], layout=layout) else: for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) sorted_params = sorted(list(set(params))) padding_ratio = 0.05 param_values_range = {} for p_name in sorted_params: values = [t.params[p_name] for t in trials if p_name in t.params] max_value = max(values) min_value = min(values) if _is_log_scale(trials, p_name): padding = (math.log10(max_value) - math.log10(min_value)) * padding_ratio min_value = math.pow(10, math.log10(min_value) - padding) max_value = math.pow(10, math.log10(max_value) + padding) else: padding = (max_value - min_value) * padding_ratio min_value = min_value - padding max_value = max_value + padding param_values_range[p_name] = (min_value, max_value) if len(sorted_params) == 2: x_param = sorted_params[0] y_param = sorted_params[1] sub_plots = _generate_contour_subplot(trials, x_param, y_param, study.direction, param_values_range) figure = go.Figure(data=sub_plots, layout=layout) figure.update_xaxes(title_text=x_param, range=param_values_range[x_param]) figure.update_yaxes(title_text=y_param, range=param_values_range[y_param]) if _is_log_scale(trials, x_param): log_range = [math.log10(p) for p in param_values_range[x_param]] figure.update_xaxes(range=log_range, type="log") if _is_log_scale(trials, y_param): log_range = [math.log10(p) for p in param_values_range[y_param]] figure.update_yaxes(range=log_range, type="log") else: figure = make_subplots(rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True) figure.update_layout(layout) showscale = True # showscale option only needs to be specified once for x_i, x_param in enumerate(sorted_params): for y_i, y_param in enumerate(sorted_params): if x_param == y_param: figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1) else: sub_plots = _generate_contour_subplot( trials, x_param, y_param, study.direction, param_values_range) contour = sub_plots[0] scatter = sub_plots[1] contour.update( showscale=showscale) # showscale's default is True if showscale: showscale = False figure.add_trace(contour, row=y_i + 1, col=x_i + 1) figure.add_trace(scatter, row=y_i + 1, col=x_i + 1) figure.update_xaxes(range=param_values_range[x_param], row=y_i + 1, col=x_i + 1) figure.update_yaxes(range=param_values_range[y_param], row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, x_param): log_range = [ math.log10(p) for p in param_values_range[x_param] ] figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, y_param): log_range = [ math.log10(p) for p in param_values_range[y_param] ] figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if x_i == 0: figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1) if y_i == len(sorted_params) - 1: figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1) return figure
def plot_pareto_front( study: Study, *, target_names: Optional[List[str]] = None, include_dominated_trials: bool = True, axis_order: Optional[List[int]] = None, constraints_func: Optional[Callable[[FrozenTrial], Sequence[float]]] = None, ) -> "go.Figure": """Plot the Pareto front of a study. Example: The following code snippet shows how to plot the Pareto front of a study. .. plotly:: import optuna def objective(trial): x = trial.suggest_float("x", 0, 5) y = trial.suggest_float("y", 0, 3) v0 = 4 * x ** 2 + 4 * y ** 2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 study = optuna.create_study(directions=["minimize", "minimize"]) study.optimize(objective, n_trials=50) fig = optuna.visualization.plot_pareto_front(study) fig.show() Args: study: A :class:`~optuna.study.Study` object whose trials are plotted for their objective values. target_names: Objective name list used as the axis titles. If :obj:`None` is specified, "Objective {objective_index}" is used instead. include_dominated_trials: A flag to include all dominated trial's objective values. axis_order: A list of indices indicating the axis order. If :obj:`None` is specified, default order is used. constraints_func: An optional function that computes the objective constraints. It must take a :class:`~optuna.trial.FrozenTrial` and return the constraints. The return value must be a sequence of :obj:`float` s. A value strictly larger than 0 means that a constraint is violated. A value equal to or smaller than 0 is considered feasible. This specification is the same as in, for example, :class:`~optuna.integration.NSGAIISampler`. If given, trials are classified into three categories: feasible and best, feasible but non-best, and infeasible. Categories are shown in different colors. Here, whether a trial is best (on Pareto front) or not is determined ignoring all infeasible trials. Returns: A :class:`plotly.graph_objs.Figure` object. Raises: :exc:`ValueError`: If the number of objectives of ``study`` isn't 2 or 3. """ _imports.check() n_dim = len(study.directions) if n_dim not in (2, 3): raise ValueError( "`plot_pareto_front` function only supports 2 or 3 objective studies." ) if target_names is None: target_names = [f"Objective {i}" for i in range(n_dim)] elif len(target_names) != n_dim: raise ValueError( f"The length of `target_names` is supposed to be {n_dim}.") if constraints_func is not None: feasible_trials = [] infeasible_trials = [] for trial in study.get_trials(states=(TrialState.COMPLETE, )): if all(map(lambda x: x <= 0.0, constraints_func(trial))): feasible_trials.append(trial) else: infeasible_trials.append(trial) best_trials = _get_pareto_front_trials_by_trials( feasible_trials, study.directions) if include_dominated_trials: non_best_trials = _get_non_pareto_front_trials( feasible_trials, best_trials) else: non_best_trials = [] if len(best_trials) == 0: _logger.warning( "Your study does not have any completed and feasible trials.") else: best_trials = study.best_trials if len(best_trials) == 0: _logger.warning("Your study does not have any completed trials.") if include_dominated_trials: non_best_trials = _get_non_pareto_front_trials( study.get_trials(deepcopy=False), best_trials) else: non_best_trials = [] infeasible_trials = [] if axis_order is None: axis_order = list(range(n_dim)) else: if len(axis_order) != n_dim: raise ValueError( f"Size of `axis_order` {axis_order}. Expect: {n_dim}, Actual: {len(axis_order)}." ) if len(set(axis_order)) != n_dim: raise ValueError( f"Elements of given `axis_order` {axis_order} are not unique!." ) if max(axis_order) > n_dim - 1: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {max(axis_order)} " f"higher than {n_dim - 1}.") if min(axis_order) < 0: raise ValueError( f"Given `axis_order` {axis_order} contains invalid index {min(axis_order)} " "lower than 0.") def _make_scatter_object( trials: Sequence[FrozenTrial], hovertemplate: str, infeasible: bool = False, dominated_trials: bool = False, ) -> Union["go.Scatter", "go.Scatter3d"]: return _make_scatter_object_base( n_dim, trials, axis_order, # type: ignore include_dominated_trials, hovertemplate=hovertemplate, infeasible=infeasible, dominated_trials=dominated_trials, ) if constraints_func is None: data = [ _make_scatter_object( non_best_trials, hovertemplate="%{text}<extra>Trial</extra>", dominated_trials=True, ), _make_scatter_object( best_trials, hovertemplate="%{text}<extra>Best Trial</extra>", dominated_trials=False, ), ] else: data = [ _make_scatter_object( infeasible_trials, hovertemplate="%{text}<extra>Infeasible Trial</extra>", infeasible=True, ), _make_scatter_object( non_best_trials, hovertemplate="%{text}<extra>Feasible Trial</extra>", dominated_trials=True, ), _make_scatter_object( best_trials, hovertemplate="%{text}<extra>Best Trial</extra>", dominated_trials=False, ), ] if n_dim == 2: layout = go.Layout( title="Pareto-front Plot", xaxis_title=target_names[axis_order[0]], yaxis_title=target_names[axis_order[1]], ) else: layout = go.Layout( title="Pareto-front Plot", scene={ "xaxis_title": target_names[axis_order[0]], "yaxis_title": target_names[axis_order[1]], "zaxis_title": target_names[axis_order[2]], }, ) return go.Figure(data=data, layout=layout)