def test_filter_inf_trials_message(caplog: LogCaptureFixture, with_message: bool) -> None: study = create_study() study.add_trial( create_trial( value=0.0, params={"x": 1.0}, distributions={"x": FloatDistribution(0.0, 1.0)}, )) study.add_trial( create_trial( value=float("inf"), params={"x": 0.0}, distributions={"x": FloatDistribution(0.0, 1.0)}, )) optuna.logging.enable_propagation() _filter_nonfinite(study.get_trials(states=(TrialState.COMPLETE, )), with_message=with_message) msg = "Trial 1 is omitted in visualization because its objective value is inf or nan." if with_message: assert msg in caplog.text n_filtered_as_inf = 0 for record in caplog.records: if record.msg == msg: assert record.levelno == logging.WARNING n_filtered_as_inf += 1 assert n_filtered_as_inf == 1 else: assert msg not in caplog.text
def test_filter_inf_trials_multiobjective(value: float, objective_selected: int, expected: int) -> None: study = create_study(directions=["minimize", "maximize"]) study.add_trial( create_trial( values=[0.0, 1.0], params={"x": 1.0}, distributions={"x": FloatDistribution(0.0, 1.0)}, )) study.add_trial( create_trial( values=[0.0, value], params={"x": 0.0}, distributions={"x": FloatDistribution(0.0, 1.0)}, )) study.add_trial( create_trial( values=[value, value], params={"x": 0.0}, distributions={"x": FloatDistribution(0.0, 1.0)}, )) def _target(t: FrozenTrial) -> float: return t.values[objective_selected] trials = _filter_nonfinite( study.get_trials(states=(TrialState.COMPLETE, )), target=_target) assert len(trials) == expected assert all([t.number == num for t, num in zip(trials, range(expected))])
def _get_edf_plot( studies: List[Study], target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "Axes": # Set up the graph style. plt.style.use( "ggplot") # Use ggplot style sheet for similar outputs to plotly. _, ax = plt.subplots() ax.set_title("Empirical Distribution Function Plot") ax.set_xlabel(target_name) ax.set_ylabel("Cumulative Probability") ax.set_ylim(0, 1) cmap = plt.get_cmap("tab20") # Use tab20 colormap for multiple line plots. # Prepare data for plotting. if len(studies) == 0: _logger.warning("There are no studies.") return ax if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target all_values: List[np.ndarray] = [] for study in studies: trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) values = np.array([target(trial) for trial in trials]) all_values.append(values) if all(len(values) == 0 for values in all_values): _logger.warning("There are no complete trials.") return ax min_x_value = np.min(np.concatenate(all_values)) max_x_value = np.max(np.concatenate(all_values)) x_values = np.linspace(min_x_value, max_x_value, 100) # Draw multiple line plots. for i, (values, study) in enumerate(zip(all_values, studies)): y_values = np.sum(values[:, np.newaxis] <= x_values, axis=0) / values.size ax.plot(x_values, y_values, color=cmap(i), alpha=0.7, label=study.study_name) if len(studies) >= 2: ax.legend() return ax
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params) if len(distributions) == 0: return OrderedDict() trials = [] for trial in _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target): if any(name not in trial.params for name in distributions.keys()): continue trials.append(trial) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) n_trials = len(trials) self._trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64) self._trans_values = numpy.empty(n_trials, dtype=numpy.float64) for trial_idx, trial in enumerate(trials): self._trans_params[trial_idx] = trans.transform(trial.params) self._trans_values[ trial_idx] = trial.value if target is None else target(trial) encoded_column_to_column = trans.encoded_column_to_column if self._trans_params.size == 0: # `params` were given but as an empty list. return OrderedDict() forest = self._forest forest.fit(self._trans_params, self._trans_values) feature_importances = forest.feature_importances_ feature_importances_reduced = numpy.zeros(len(distributions)) numpy.add.at(feature_importances_reduced, encoded_column_to_column, feature_importances) param_importances = OrderedDict() self._param_names = list(distributions.keys()) for i in feature_importances_reduced.argsort()[::-1]: param_importances[ self._param_names[i]] = feature_importances_reduced[i].item() return param_importances
def _get_contour_info( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> _ContourInfo: _check_plot_args(study, target, target_name) trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) all_params = {p_name for t in trials for p_name in t.params.keys()} if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") sorted_params = [] elif params is None: sorted_params = sorted(all_params) else: if len(params) <= 1: _logger.warning("The length of params must be greater than 1.") for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) sorted_params = sorted(set(params)) sub_plot_infos: List[List[_SubContourInfo]] if len(sorted_params) == 2: x_param = sorted_params[0] y_param = sorted_params[1] sub_plot_info = _get_contour_subplot_info(trials, x_param, y_param, target) sub_plot_infos = [[sub_plot_info]] else: sub_plot_infos = [] for i, y_param in enumerate(sorted_params): sub_plot_infos.append([]) for x_param in sorted_params: sub_plot_info = _get_contour_subplot_info( trials, x_param, y_param, target) sub_plot_infos[i].append(sub_plot_info) reverse_scale = _is_reverse_scale(study, target) return _ContourInfo( sorted_params=sorted_params, sub_plot_infos=sub_plot_infos, reverse_scale=reverse_scale, target_name=target_name, )
def _get_edf_info( study: Union[Study, Sequence[Study]], target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> _EDFInfo: if isinstance(study, Study): studies = [study] else: studies = list(study) _check_plot_args(studies, target, target_name) if len(studies) == 0: _logger.warning("There are no studies.") return _EDFInfo(lines=[], x_values=np.array([])) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target study_names = [] all_values: List[np.ndarray] = [] for study in studies: trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) values = np.array([target(trial) for trial in trials]) all_values.append(values) study_names.append(study.study_name) if all(len(values) == 0 for values in all_values): _logger.warning("There are no complete trials.") return _EDFInfo(lines=[], x_values=np.array([])) min_x_value = np.min(np.concatenate(all_values)) max_x_value = np.max(np.concatenate(all_values)) x_values = np.linspace(min_x_value, max_x_value, NUM_SAMPLES_X_AXIS) edf_line_info_list = [] for (study_name, values) in zip(study_names, all_values): y_values = np.sum(values[:, np.newaxis] <= x_values, axis=0) / values.size edf_line_info_list.append( _EDFLineInfo(study_name=study_name, y_values=y_values)) return _EDFInfo(lines=edf_line_info_list, x_values=x_values)
def _get_edf_plot( studies: List[Study], target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout( title="Empirical Distribution Function Plot", xaxis={"title": target_name}, yaxis={"title": "Cumulative Probability"}, ) if len(studies) == 0: _logger.warning("There are no studies.") return go.Figure(data=[], layout=layout) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target all_values: List[np.ndarray] = [] for study in studies: trials = _filter_nonfinite( study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target ) values = np.array([target(trial) for trial in trials]) all_values.append(values) if all(len(values) == 0 for values in all_values): _logger.warning("There are no complete trials.") return go.Figure(data=[], layout=layout) min_x_value = np.min(np.concatenate(all_values)) max_x_value = np.max(np.concatenate(all_values)) x_values = np.linspace(min_x_value, max_x_value, 100) traces = [] for values, study in zip(all_values, studies): y_values = np.sum(values[:, np.newaxis] <= x_values, axis=0) / values.size traces.append(go.Scatter(x=x_values, y=y_values, name=study.study_name, mode="lines")) figure = go.Figure(data=traces, layout=layout) figure.update_yaxes(range=[0, 1]) return figure
def test_filter_inf_trials(value: float, expected: int) -> None: study = create_study() study.add_trial( create_trial( value=0.0, params={"x": 1.0}, distributions={"x": FloatDistribution(0.0, 1.0)}, )) study.add_trial( create_trial( value=value, params={"x": 0.0}, distributions={"x": FloatDistribution(0.0, 1.0)}, )) trials = _filter_nonfinite( study.get_trials(states=(TrialState.COMPLETE, ))) assert len(trials) == expected assert all([t.number == num for t, num in zip(trials, range(expected))])
def _get_importances_info( study: Study, evaluator: Optional[BaseImportanceEvaluator], params: Optional[List[str]], target: Optional[Callable[[FrozenTrial], float]], target_name: str, ) -> _ImportancesInfo: _check_plot_args(study, target, target_name) trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) if len(trials) == 0: logger.warning("Study instance does not contain completed trials.") return _ImportancesInfo( importance_values=[], param_names=[], importance_labels=[], target_name=target_name, ) importances = optuna.importance.get_param_importances(study, evaluator=evaluator, params=params, target=target) importances = OrderedDict(reversed(list(importances.items()))) importance_values = list(importances.values()) param_names = list(importances.keys()) importance_labels = [ f"{val:.2f}" if val >= 0.01 else "<0.01" for val in importance_values ] return _ImportancesInfo( importance_values=importance_values, param_names=param_names, importance_labels=importance_labels, target_name=target_name, )
def _get_slice_plot_info( study: Study, params: Optional[List[str]], target: Optional[Callable[[FrozenTrial], float]], target_name: str, ) -> _SlicePlotInfo: _check_plot_args(study, target, target_name) trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return _SlicePlotInfo(target_name, []) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(all_params) else: for input_p_name in params: if input_p_name not in all_params: raise ValueError( f"Parameter {input_p_name} does not exist in your study.") sorted_params = sorted(set(params)) return _SlicePlotInfo( target_name=target_name, subplots=[ _get_slice_subplot_info( trials=trials, param=param, target=target, log_scale=_is_log_scale(trials, param), numerical=_is_numerical(trials, param), ) for param in sorted_params ], )
def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params) if len(distributions) == 0: return OrderedDict() # fANOVA does not support parameter distributions with a single value. # However, there is no reason to calculate parameter importance in such case anyway, # since it will always be 0 as the parameter is constant in the objective function. zero_importances = { name: 0.0 for name, dist in distributions.items() if dist.single() } distributions = { name: dist for name, dist in distributions.items() if not dist.single() } trials = [] for trial in _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target): if any(name not in trial.params for name in distributions.keys()): continue trials.append(trial) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) n_trials = len(trials) trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64) trans_values = numpy.empty(n_trials, dtype=numpy.float64) for trial_idx, trial in enumerate(trials): trans_params[trial_idx] = trans.transform(trial.params) trans_values[ trial_idx] = trial.value if target is None else target(trial) trans_bounds = trans.bounds column_to_encoded_columns = trans.column_to_encoded_columns if trans_params.size == 0: # `params` were given but as an empty list. return OrderedDict() # Many (deep) copies of the search spaces are required during the tree traversal and using # Optuna distributions will create a bottleneck. # Therefore, search spaces (parameter distributions) are represented by a single # `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical # or not. evaluator = self._evaluator evaluator.fit( X=trans_params, y=trans_values, search_spaces=trans_bounds, column_to_encoded_columns=column_to_encoded_columns, ) importances = {} for i, name in enumerate(distributions.keys()): importance, _ = evaluator.get_importance((i, )) importances[name] = importance importances = {**importances, **zero_importances} total_importance = sum(importances.values()) for name in importances: importances[name] /= total_importance sorted_importances = OrderedDict( reversed( sorted( importances.items(), key=lambda name_and_importance: name_and_importance[1]))) return sorted_importances
def _get_parallel_coordinate_info( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> _ParallelCoordinateInfo: reverse_scale = _is_reverse_scale(study, target) trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is not None: for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) all_params = set(params) sorted_params = sorted(all_params) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target skipped_trial_numbers = _get_skipped_trial_numbers(trials, sorted_params) objectives = tuple( [target(t) for t in trials if t.number not in skipped_trial_numbers]) # The value of (0, 0) is a dummy range. It is ignored when we plot. objective_range = (min(objectives), max(objectives)) if len(objectives) > 0 else (0, 0) dim_objective = _DimensionInfo( label=target_name, values=objectives, range=objective_range, is_log=False, is_cat=False, tickvals=[], ticktext=[], ) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return _ParallelCoordinateInfo( dim_objective=dim_objective, dims_params=[], reverse_scale=reverse_scale, target_name=target_name, ) if len(objectives) == 0: _logger.warning( "Your study has only completed trials with missing parameters.") return _ParallelCoordinateInfo( dim_objective=dim_objective, dims_params=[], reverse_scale=reverse_scale, target_name=target_name, ) numeric_cat_params_indices: List[int] = [] dims = [] for dim_index, p_name in enumerate(sorted_params, start=1): values = [] for t in trials: if t.number in skipped_trial_numbers: continue if p_name in t.params: values.append(t.params[p_name]) if _is_log_scale(trials, p_name): values = [math.log10(v) for v in values] min_value = min(values) max_value = max(values) tickvals = list(range(math.ceil(min_value), math.ceil(max_value))) if min_value not in tickvals: tickvals = [min_value] + tickvals if max_value not in tickvals: tickvals = tickvals + [max_value] dim = _DimensionInfo( label=_truncate_label(p_name), values=tuple(values), range=(min_value, max_value), is_log=True, is_cat=False, tickvals=tickvals, ticktext=["{:.3g}".format(math.pow(10, x)) for x in tickvals], ) elif _is_categorical(trials, p_name): vocab: DefaultDict[str, int] = defaultdict(lambda: len(vocab)) if _is_numerical(trials, p_name): _ = [vocab[v] for v in sorted(values)] values = [vocab[v] for v in values] ticktext = list(sorted(vocab.keys())) numeric_cat_params_indices.append(dim_index) else: values = [vocab[v] for v in values] ticktext = list(sorted(vocab.keys(), key=lambda x: vocab[x])) dim = _DimensionInfo( label=_truncate_label(p_name), values=tuple(values), range=(min(values), max(values)), is_log=False, is_cat=True, tickvals=list(range(len(vocab))), ticktext=ticktext, ) else: dim = _DimensionInfo( label=_truncate_label(p_name), values=tuple(values), range=(min(values), max(values)), is_log=False, is_cat=False, tickvals=[], ticktext=[], ) dims.append(dim) if numeric_cat_params_indices: dims.insert(0, dim_objective) # np.lexsort consumes the sort keys the order from back to front. # So the values of parameters have to be reversed the order. idx = np.lexsort( [dims[index].values for index in numeric_cat_params_indices][::-1]) updated_dims = [] for dim in dims: # Since the values are mapped to other categories by the index, # the index will be swapped according to the sorted index of numeric params. updated_dims.append( _DimensionInfo( label=dim.label, values=tuple(np.array(dim.values)[idx]), range=dim.range, is_log=dim.is_log, is_cat=dim.is_cat, tickvals=dim.tickvals, ticktext=dim.ticktext, )) dim_objective = updated_dims[0] dims = updated_dims[1:] return _ParallelCoordinateInfo( dim_objective=dim_objective, dims_params=dims, reverse_scale=reverse_scale, target_name=target_name, )
def _get_parallel_coordinate_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout(title="Parallel Coordinate Plot") reverse_scale = _is_reverse_scale(study, target) trials = _filter_nonfinite( study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target ) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is not None: for input_p_name in params: if input_p_name not in all_params: raise ValueError("Parameter {} does not exist in your study.".format(input_p_name)) all_params = set(params) sorted_params = sorted(all_params) if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target skipped_trial_ids = _get_skipped_trial_numbers(trials, sorted_params) objectives = tuple([target(t) for t in trials if t.number not in skipped_trial_ids]) if len(objectives) == 0: _logger.warning("Your study has only completed trials with missing parameters.") return go.Figure(data=[], layout=layout) dims: List[Dict[str, Any]] = [ { "label": target_name, "values": objectives, "range": (min(objectives), max(objectives)), } ] numeric_cat_params_indices: List[int] = [] for dim_index, p_name in enumerate(sorted_params, start=1): values = [] for t in trials: if t.number in skipped_trial_ids: continue if p_name in t.params: values.append(t.params[p_name]) if _is_log_scale(trials, p_name): values = [math.log10(v) for v in values] min_value = min(values) max_value = max(values) tickvals = list(range(math.ceil(min_value), math.ceil(max_value))) if min_value not in tickvals: tickvals = [min_value] + tickvals if max_value not in tickvals: tickvals = tickvals + [max_value] dim = { "label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]), "values": tuple(values), "range": (min_value, max_value), "tickvals": tickvals, "ticktext": ["{:.3g}".format(math.pow(10, x)) for x in tickvals], } elif _is_categorical(trials, p_name): vocab: DefaultDict[str, int] = defaultdict(lambda: len(vocab)) if _is_numerical(trials, p_name): _ = [vocab[v] for v in sorted(values)] values = [vocab[v] for v in values] ticktext = list(sorted(vocab.keys())) numeric_cat_params_indices.append(dim_index) else: values = [vocab[v] for v in values] ticktext = list(sorted(vocab.keys(), key=lambda x: vocab[x])) dim = { "label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]), "values": tuple(values), "range": (min(values), max(values)), "tickvals": list(range(len(vocab))), "ticktext": ticktext, } else: dim = { "label": p_name if len(p_name) < 20 else "{}...".format(p_name[:17]), "values": tuple(values), "range": (min(values), max(values)), } dims.append(dim) if numeric_cat_params_indices: # np.lexsort consumes the sort keys the order from back to front. # So the values of parameters have to be reversed the order. idx = np.lexsort([dims[index]["values"] for index in numeric_cat_params_indices][::-1]) for dim in dims: # Since the values are mapped to other categories by the index, # the index will be swapped according to the sorted index of numeric params. dim.update({"values": tuple(np.array(dim["values"])[idx])}) traces = [ go.Parcoords( dimensions=dims, labelangle=30, labelside="bottom", line={ "color": dims[0]["values"], "colorscale": COLOR_SCALE, "colorbar": {"title": target_name}, "showscale": True, "reversescale": reverse_scale, }, ) ] figure = go.Figure(data=traces, layout=layout) return figure
def plot_param_importances( study: Study, evaluator: Optional[BaseImportanceEvaluator] = None, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": """Plot hyperparameter importances. Example: The following code snippet shows how to plot hyperparameter importances. .. plotly:: import optuna def objective(trial): x = trial.suggest_int("x", 0, 2) y = trial.suggest_float("y", -1.0, 1.0) z = trial.suggest_float("z", 0.0, 1.5) return x ** 2 + y ** 3 - z ** 4 sampler = optuna.samplers.RandomSampler(seed=10) study = optuna.create_study(sampler=sampler) study.optimize(objective, n_trials=100) fig = optuna.visualization.plot_param_importances(study) fig.show() .. seealso:: This function visualizes the results of :func:`optuna.importance.get_param_importances`. Args: study: An optimized study. evaluator: An importance evaluator object that specifies which algorithm to base the importance assessment on. Defaults to :class:`~optuna.importance.FanovaImportanceEvaluator`. params: A list of names of parameters to assess. If :obj:`None`, all parameters that are present in all of the completed trials are assessed. target: A function to specify the value to display. If it is :obj:`None` and ``study`` is being used for single-objective optimization, the objective values are plotted. .. note:: Specify this argument if ``study`` is being used for multi-objective optimization. For example, to get the hyperparameter importance of the first objective, use ``target=lambda t: t.values[0]`` for the target parameter. target_name: Target's name to display on the axis label. Returns: A :class:`plotly.graph_objs.Figure` object. """ _imports.check() _check_plot_args(study, target, target_name) layout = go.Layout( title="Hyperparameter Importances", xaxis={"title": f"Importance for {target_name}"}, yaxis={"title": "Hyperparameter"}, showlegend=False, ) # Importances cannot be evaluated without completed trials. # Return an empty figure for consistency with other visualization functions. trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) if len(trials) == 0: logger.warning("Study instance does not contain completed trials.") return go.Figure(data=[], layout=layout) importances = optuna.importance.get_param_importances(study, evaluator=evaluator, params=params, target=target) importances = OrderedDict(reversed(list(importances.items()))) importance_values = list(importances.values()) param_names = list(importances.keys()) importance_labels = [ f"{val:.2f}" if val >= 0.01 else "<0.01" for val in importance_values ] fig = go.Figure( data=[ go.Bar( x=importance_values, y=param_names, text=importance_labels, textposition="outside", cliponaxis=False, # Ensure text is not clipped. hovertemplate=[ _make_hovertext(param_name, importance, study) for param_name, importance in importances.items() ], marker_color=plotly.colors.sequential.Blues[-4], orientation="h", ) ], layout=layout, ) return fig
def _get_param_importance_plot( study: Study, evaluator: Optional[BaseImportanceEvaluator] = None, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "Axes": # Set up the graph style. plt.style.use( "ggplot") # Use ggplot style sheet for similar outputs to plotly. fig, ax = plt.subplots() ax.set_title("Hyperparameter Importances") ax.set_xlabel(f"Importance for {target_name}") ax.set_ylabel("Hyperparameter") # Prepare data for plotting. # Importances cannot be evaluated without completed trials. # Return an empty figure for consistency with other visualization functions. trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) if len(trials) == 0: _logger.warning("Study instance does not contain completed trials.") return ax importances = optuna.importance.get_param_importances(study, evaluator=evaluator, params=params, target=target) importances = OrderedDict(reversed(list(importances.items()))) importance_values = list(importances.values()) param_names = list(importances.keys()) pos = np.arange(len(param_names)) # Draw horizontal bars. ax.barh( pos, importance_values, align="center", color=cm.get_cmap("tab20c")(0), tick_label=param_names, ) renderer = fig.canvas.get_renderer() for idx, val in enumerate(importance_values): label = f" {val:.2f}" if val >= 0.01 else " <0.01" text = ax.text(val, idx, label, va="center") # Sometimes horizontal axis needs to be re-scaled # to avoid text going over plot area. bbox = text.get_window_extent(renderer) bbox = bbox.transformed(ax.transData.inverted()) _, plot_xmax = ax.get_xlim() bbox_xmax = bbox.xmax if bbox_xmax > plot_xmax: ax.set_xlim(xmax=AXES_PADDING_RATIO * bbox_xmax) return ax
def test_filter_nonfinite_with_invalid_target() -> None: study = prepare_study_with_trials() trials = study.get_trials(states=(TrialState.COMPLETE,)) with pytest.raises(ValueError): _filter_nonfinite(trials, target=lambda t: "invalid target") # type: ignore
def _get_contour_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "Axes": # Calculate basic numbers for plotting. trials = _filter_nonfinite( study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target ) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") _, ax = plt.subplots() return ax all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(all_params) elif len(params) <= 1: _logger.warning("The length of params must be greater than 1.") _, ax = plt.subplots() return ax else: for input_p_name in params: if input_p_name not in all_params: raise ValueError("Parameter {} does not exist in your study.".format(input_p_name)) sorted_params = sorted(set(params)) n_params = len(sorted_params) plt.style.use("ggplot") # Use ggplot style sheet for similar outputs to plotly. if n_params == 2: # Set up the graph style. fig, axs = plt.subplots() axs.set_title("Contour Plot") cmap = _set_cmap(study, target) contour_point_num = 100 # Prepare data and draw contour plots. if params: x_param = params[0] y_param = params[1] else: x_param = sorted_params[0] y_param = sorted_params[1] cs = _generate_contour_subplot( trials, x_param, y_param, axs, cmap, contour_point_num, target ) if isinstance(cs, ContourSet): axcb = fig.colorbar(cs) axcb.set_label(target_name) else: # Set up the graph style. fig, axs = plt.subplots(n_params, n_params) fig.suptitle("Contour Plot") cmap = _set_cmap(study, target) contour_point_num = 100 # Prepare data and draw contour plots. cs_list = [] for x_i, x_param in enumerate(sorted_params): for y_i, y_param in enumerate(sorted_params): ax = axs[y_i, x_i] cs = _generate_contour_subplot( trials, x_param, y_param, ax, cmap, contour_point_num, target ) if isinstance(cs, ContourSet): cs_list.append(cs) if cs_list: axcb = fig.colorbar(cs_list[0], ax=axs) axcb.set_label(target_name) return axs
def _get_contour_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "go.Figure": layout = go.Layout(title="Contour Plot") trials = _filter_nonfinite(study.get_trials( deepcopy=False, states=(TrialState.COMPLETE, )), target=target) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return go.Figure(data=[], layout=layout) all_params = {p_name for t in trials for p_name in t.params.keys()} if params is None: sorted_params = sorted(all_params) elif len(params) <= 1: _logger.warning("The length of params must be greater than 1.") return go.Figure(data=[], layout=layout) else: for input_p_name in params: if input_p_name not in all_params: raise ValueError( "Parameter {} does not exist in your study.".format( input_p_name)) sorted_params = sorted(set(params)) padding_ratio = 0.05 param_values_range = {} for p_name in sorted_params: values = _get_param_values(trials, p_name) min_value = min(values) max_value = max(values) if _is_log_scale(trials, p_name): padding = (math.log10(max_value) - math.log10(min_value)) * padding_ratio min_value = math.pow(10, math.log10(min_value) - padding) max_value = math.pow(10, math.log10(max_value) + padding) elif _is_numerical(trials, p_name): padding = (max_value - min_value) * padding_ratio min_value = min_value - padding max_value = max_value + padding else: # Plotly>=4.12.0 draws contours using the indices of categorical variables instead of # raw values and the range should be updated based on the cardinality of categorical # variables. See https://github.com/optuna/optuna/issues/1967. if version.parse(plotly.__version__) >= version.parse("4.12.0"): span = len(set(values)) - 1 padding = span * padding_ratio min_value = -padding max_value = span + padding param_values_range[p_name] = (min_value, max_value) reverse_scale = _is_reverse_scale(study, target) if len(sorted_params) == 2: x_param = sorted_params[0] y_param = sorted_params[1] sub_plots = _generate_contour_subplot(trials, x_param, y_param, reverse_scale, param_values_range, target, target_name) figure = go.Figure(data=sub_plots, layout=layout) figure.update_xaxes(title_text=x_param, range=param_values_range[x_param]) figure.update_yaxes(title_text=y_param, range=param_values_range[y_param]) if not _is_numerical(trials, x_param): figure.update_xaxes(type="category") if not _is_numerical(trials, y_param): figure.update_yaxes(type="category") if _is_log_scale(trials, x_param): log_range = [math.log10(p) for p in param_values_range[x_param]] figure.update_xaxes(range=log_range, type="log") if _is_log_scale(trials, y_param): log_range = [math.log10(p) for p in param_values_range[y_param]] figure.update_yaxes(range=log_range, type="log") else: figure = make_subplots(rows=len(sorted_params), cols=len(sorted_params), shared_xaxes=True, shared_yaxes=True) figure.update_layout(layout) showscale = True # showscale option only needs to be specified once for x_i, x_param in enumerate(sorted_params): for y_i, y_param in enumerate(sorted_params): if x_param == y_param: figure.add_trace(go.Scatter(), row=y_i + 1, col=x_i + 1) else: sub_plots = _generate_contour_subplot( trials, x_param, y_param, reverse_scale, param_values_range, target, target_name, ) contour = sub_plots[0] scatter = sub_plots[1] contour.update( showscale=showscale) # showscale's default is True if showscale: showscale = False figure.add_trace(contour, row=y_i + 1, col=x_i + 1) figure.add_trace(scatter, row=y_i + 1, col=x_i + 1) figure.update_xaxes(range=param_values_range[x_param], row=y_i + 1, col=x_i + 1) figure.update_yaxes(range=param_values_range[y_param], row=y_i + 1, col=x_i + 1) if not _is_numerical(trials, x_param): figure.update_xaxes(type="category", row=y_i + 1, col=x_i + 1) if not _is_numerical(trials, y_param): figure.update_yaxes(type="category", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, x_param): log_range = [ math.log10(p) for p in param_values_range[x_param] ] figure.update_xaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if _is_log_scale(trials, y_param): log_range = [ math.log10(p) for p in param_values_range[y_param] ] figure.update_yaxes(range=log_range, type="log", row=y_i + 1, col=x_i + 1) if x_i == 0: figure.update_yaxes(title_text=y_param, row=y_i + 1, col=x_i + 1) if y_i == len(sorted_params) - 1: figure.update_xaxes(title_text=x_param, row=y_i + 1, col=x_i + 1) return figure
def _get_parallel_coordinate_plot( study: Study, params: Optional[List[str]] = None, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ) -> "Axes": if target is None: def _target(t: FrozenTrial) -> float: return cast(float, t.value) target = _target reversescale = study.direction == StudyDirection.MINIMIZE else: reversescale = True # Set up the graph style. fig, ax = plt.subplots() cmap = plt.get_cmap("Blues_r" if reversescale else "Blues") ax.set_title("Parallel Coordinate Plot") ax.spines["top"].set_visible(False) ax.spines["bottom"].set_visible(False) # Prepare data for plotting. trials = _filter_nonfinite( study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,)), target=target ) if len(trials) == 0: _logger.warning("Your study does not have any completed trials.") return ax all_params = {p_name for t in trials for p_name in t.params.keys()} if params is not None: for input_p_name in params: if input_p_name not in all_params: raise ValueError("Parameter {} does not exist in your study.".format(input_p_name)) all_params = set(params) sorted_params = sorted(all_params) skipped_trial_numbers = _get_skipped_trial_numbers(trials, sorted_params) obj_org = [target(t) for t in trials if t.number not in skipped_trial_numbers] if len(obj_org) == 0: _logger.warning("Your study has only completed trials with missing parameters.") return ax obj_min = min(obj_org) obj_max = max(obj_org) obj_w = obj_max - obj_min dims_obj_base = [[o] for o in obj_org] cat_param_names = [] cat_param_values = [] cat_param_ticks = [] param_values = [] var_names = [target_name] numeric_cat_params_indices: List[int] = [] for param_index, p_name in enumerate(sorted_params): values = [t.params[p_name] for t in trials if t.number not in skipped_trial_numbers] if _is_categorical(trials, p_name): vocab = defaultdict(lambda: len(vocab)) # type: DefaultDict[str, int] if _is_numerical(trials, p_name): _ = [vocab[v] for v in sorted(values)] numeric_cat_params_indices.append(param_index) values = [vocab[v] for v in values] cat_param_names.append(p_name) vocab_item_sorted = sorted(vocab.items(), key=lambda x: x[1]) cat_param_values.append([v[0] for v in vocab_item_sorted]) cat_param_ticks.append([v[1] for v in vocab_item_sorted]) if _is_log_scale(trials, p_name): values_for_lc = [np.log10(v) for v in values] else: values_for_lc = values p_min = min(values_for_lc) p_max = max(values_for_lc) p_w = p_max - p_min if p_w == 0.0: center = obj_w / 2 + obj_min for i in range(len(values)): dims_obj_base[i].append(center) else: for i, v in enumerate(values_for_lc): dims_obj_base[i].append((v - p_min) / p_w * obj_w + obj_min) var_names.append(p_name if len(p_name) < 20 else "{}...".format(p_name[:17])) param_values.append(values) if numeric_cat_params_indices: # np.lexsort consumes the sort keys the order from back to front. # So the values of parameters have to be reversed the order. sorted_idx = np.lexsort( [param_values[index] for index in numeric_cat_params_indices][::-1] ) # Since the values are mapped to other categories by the index, # the index will be swapped according to the sorted index of numeric params. param_values = [list(np.array(v)[sorted_idx]) for v in param_values] # Draw multiple line plots and axes. # Ref: https://stackoverflow.com/a/50029441 ax.set_xlim(0, len(sorted_params)) ax.set_ylim(obj_min, obj_max) xs = [range(len(sorted_params) + 1) for _ in range(len(dims_obj_base))] segments = [np.column_stack([x, y]) for x, y in zip(xs, dims_obj_base)] lc = LineCollection(segments, cmap=cmap) lc.set_array(np.asarray(obj_org)) axcb = fig.colorbar(lc, pad=0.1) axcb.set_label(target_name) plt.xticks(range(len(sorted_params) + 1), var_names, rotation=330) for i, p_name in enumerate(sorted_params): ax2 = ax.twinx() ax2.set_ylim(min(param_values[i]), max(param_values[i])) if _is_log_scale(trials, p_name): ax2.set_yscale("log") ax2.spines["top"].set_visible(False) ax2.spines["bottom"].set_visible(False) ax2.xaxis.set_visible(False) ax2.spines["right"].set_position(("axes", (i + 1) / len(sorted_params))) if p_name in cat_param_names: idx = cat_param_names.index(p_name) tick_pos = cat_param_ticks[idx] tick_labels = cat_param_values[idx] ax2.set_yticks(tick_pos) ax2.set_yticklabels(tick_labels) ax.add_collection(lc) return ax