def test_constraints_func_experimental_warning() -> None: study = optuna.create_study(directions=["minimize", "minimize"]) def constraints_func(t: FrozenTrial) -> Sequence[float]: return [1.0] with pytest.warns(optuna.exceptions.ExperimentalWarning): plot_pareto_front( study=study, constraints_func=constraints_func, )
def test_plot_pareto_front_targets_without_target_names() -> None: study = optuna.create_study(directions=["minimize", "minimize", "minimize"]) with pytest.raises( ValueError, match="If `targets` is specified for empty studies, `target_names` must be specified.", ): plot_pareto_front( study=study, target_names=None, targets=lambda t: (t.values[0], t.values[1], t.values[2]), )
def test_plot_pareto_front_2d( include_dominated_trials: bool, axis_order: Optional[List[int]] ) -> None: # Test with no trial. study = optuna.create_study(directions=["minimize", "minimize"]) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.get_lines()) == 0 # Test with three trials. study.enqueue_trial({"x": 1, "y": 1}) study.enqueue_trial({"x": 1, "y": 0}) study.enqueue_trial({"x": 0, "y": 1}) study.optimize(lambda t: [t.suggest_int("x", 0, 1), t.suggest_int("y", 0, 1)], n_trials=3) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.get_lines()) == 0 # Test with `target_names` argument. with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=[], include_dominated_trials=include_dominated_trials ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo"], include_dominated_trials=include_dominated_trials ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo", "Bar", "Baz"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) target_names = ["Foo", "Bar"] figure = plot_pareto_front( study=study, target_names=target_names, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.get_lines()) == 0 if axis_order is None: assert figure.get_xlabel() == target_names[0] assert figure.get_ylabel() == target_names[1] else: assert figure.get_xlabel() == target_names[axis_order[0]] assert figure.get_ylabel() == target_names[axis_order[1]]
def test_plot_pareto_front_using_axis_order_and_targets() -> None: study = optuna.create_study(directions=["minimize", "minimize", "minimize"]) with pytest.raises( ValueError, match="Using both `targets` and `axis_order` is not supported." " Use either `targets` or `axis_order`.", ): plot_pareto_front( study=study, axis_order=[0, 1, 2], targets=lambda t: (t.values[0], t.values[1], t.values[2]), )
def test_plot_pareto_front_invalid_target_values() -> None: study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda _: [0, 0, 0, 0], n_trials=3) with pytest.raises( ValueError, match="targets` should return a sequence of target values. your `targets`" " returns <class 'float'>", ): plot_pareto_front( study=study, targets=lambda t: t.values[0], )
def test_plot_pareto_front_n_targets_unsupported( targets: Callable[[FrozenTrial], Sequence[float]] ) -> None: study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda _: [0, 0, 0, 0], n_trials=3) n_targets = len(targets(study.best_trials[0])) with pytest.raises( ValueError, match="`plot_pareto_front` function only supports 2 or 3 targets." " you used {} targets now.".format(n_targets), ): plot_pareto_front( study=study, targets=targets, )
def test_plot_pareto_front_invalid_target_values( targets: Optional[Callable[[FrozenTrial], Sequence[float]]]) -> None: study = optuna.create_study( directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda t: [0, 0, 0, 0], n_trials=3) with pytest.raises( ValueError, match= "targets` should return a sequence of target values. your `targets`" " returns <class 'float'>", ): plot_pareto_front( study=study, targets=targets, )
def test_plot_pareto_front_invalid_axis_order( dimension: int, include_dominated_trials: bool, use_constraints_func: bool ) -> None: study = optuna.create_study(directions=["minimize"] * dimension) study.optimize(lambda _: [0] * dimension, n_trials=1) constraints_func = (lambda _: [-1.0]) if use_constraints_func else None # Invalid: len(axis_order) != dimension with pytest.raises(ValueError): invalid_axis_order = list(range(dimension + 1)) assert len(invalid_axis_order) != dimension plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, ) # Invalid: np.unique(axis_order).size != dimension with pytest.raises(ValueError): invalid_axis_order = list(range(dimension)) invalid_axis_order[1] = invalid_axis_order[0] assert np.unique(invalid_axis_order).size != dimension plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, ) # Invalid: max(axis_order) > (dimension - 1) with pytest.raises(ValueError): invalid_axis_order = list(range(dimension)) invalid_axis_order[-1] += 1 assert max(invalid_axis_order) > (dimension - 1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, ) # Invalid: min(axis_order) < 0 with pytest.raises(ValueError): study = optuna.create_study(directions=["minimize", "minimize"]) study.optimize(lambda _: [0] * 2, n_trials=1) invalid_axis_order = list(range(dimension)) invalid_axis_order[0] -= 1 assert min(invalid_axis_order) < 0 plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, )
def _log_plots(run, study: optuna.Study, visualization_backend='plotly', log_plot_contour=True, log_plot_edf=True, log_plot_parallel_coordinate=True, log_plot_param_importances=True, log_plot_pareto_front=True, log_plot_slice=True, log_plot_intermediate_values=True, log_plot_optimization_history=True, ): if visualization_backend == 'matplotlib': import optuna.visualization.matplotlib as vis elif visualization_backend == 'plotly': import optuna.visualization as vis else: raise NotImplementedError(f'{visualization_backend} visualisation backend is not implemented') if vis.is_available: params = list(p_name for t in study.trials for p_name in t.params.keys()) if log_plot_contour and any(params): run['visualizations/plot_contour'] = neptune.types.File.as_html(vis.plot_contour(study)) if log_plot_edf: run['visualizations/plot_edf'] = neptune.types.File.as_html(vis.plot_edf(study)) if log_plot_parallel_coordinate: run['visualizations/plot_parallel_coordinate'] = \ neptune.types.File.as_html(vis.plot_parallel_coordinate(study)) if log_plot_param_importances and len(study.get_trials(states=(optuna.trial.TrialState.COMPLETE, optuna.trial.TrialState.PRUNED,))) > 1: try: run['visualizations/plot_param_importances'] = neptune.types.File.as_html(vis.plot_param_importances(study)) except (RuntimeError, ValueError, ZeroDivisionError): # Unable to compute importances pass if log_plot_pareto_front and study._is_multi_objective() and visualization_backend == 'plotly': run['visualizations/plot_pareto_front'] = neptune.types.File.as_html(vis.plot_pareto_front(study)) if log_plot_slice and any(params): run['visualizations/plot_slice'] = neptune.types.File.as_html(vis.plot_slice(study)) if log_plot_intermediate_values and any(trial.intermediate_values for trial in study.trials): # Intermediate values plot if available only if the above condition is met run['visualizations/plot_intermediate_values'] = \ neptune.types.File.as_html(vis.plot_intermediate_values(study)) if log_plot_optimization_history: run['visualizations/plot_optimization_history'] = \ neptune.types.File.as_html(vis.plot_optimization_history(study))
def test_plot_pareto_front_dimensions(include_dominated_trials: bool) -> None: # Unsupported: n_objectives == 1. with pytest.raises(ValueError): study = optuna.create_study(directions=["minimize"]) study.optimize(lambda t: [0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials) with pytest.raises(ValueError): study = optuna.create_study(direction="minimize") study.optimize(lambda t: [0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials) # Supported: n_objectives == 2. study = optuna.create_study(directions=["minimize", "minimize"]) study.optimize(lambda t: [0, 0], n_trials=1) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials) assert figure.has_data() # Supported: n_objectives == 3. study = optuna.create_study( directions=["minimize", "minimize", "minimize"]) study.optimize(lambda t: [0, 0, 0], n_trials=1) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials) assert figure.has_data() # Unsupported: n_objectives == 4. with pytest.raises(ValueError): study = optuna.create_study( directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda t: [0, 0, 0, 0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials)
def test_plot_pareto_front_unsupported_dimensions( include_dominated_trials: bool, use_constraints_func: bool ) -> None: constraints_func = (lambda _: [-1.0]) if use_constraints_func else None error_message = ( "`plot_pareto_front` function only supports 2 or 3 objective" " studies when using `targets` is `None`. Please use `targets`" " if your objective studies have more than 3 objectives." ) # Unsupported: n_objectives == 1. with pytest.raises(ValueError, match=error_message): study = optuna.create_study(directions=["minimize"]) study.optimize(lambda _: [0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, ) with pytest.raises(ValueError, match=error_message): study = optuna.create_study(direction="minimize") study.optimize(lambda _: [0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, ) # Unsupported: n_objectives == 4. with pytest.raises(ValueError, match=error_message): study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda _: [0, 0, 0, 0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, )
def test_plot_pareto_front_2d( include_dominated_trials: bool, axis_order: Optional[List[int]], use_constraints_func: bool, targets: Optional[Callable[[FrozenTrial], Sequence[float]]], ) -> None: if axis_order is not None and targets is not None: pytest.skip("skip using both axis_order and targets") # Test with no trial. study = optuna.create_study(directions=["minimize", "minimize"]) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.get_lines()) == 0 plt.savefig(BytesIO()) # Test with four trials. study.enqueue_trial({"x": 1, "y": 2}) study.enqueue_trial({"x": 1, "y": 1}) study.enqueue_trial({"x": 0, "y": 2}) study.enqueue_trial({"x": 1, "y": 0}) study.optimize(lambda t: [t.suggest_int("x", 0, 2), t.suggest_int("y", 0, 2)], n_trials=4) constraints_func: Optional[Callable[[FrozenTrial], Sequence[float]]] if use_constraints_func: # (x, y) = (1, 0) is infeasible; others are feasible. def constraints_func(t: FrozenTrial) -> Sequence[float]: return [1.0] if t.params["x"] == 1 and t.params["y"] == 0 else [-1.0] else: constraints_func = None figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, constraints_func=constraints_func, targets=targets, ) assert len(figure.get_lines()) == 0 actual_axis_order = axis_order or [0, 1] data: List[Tuple[int, ...]] if use_constraints_func: if include_dominated_trials: # The enqueue order of trial is: infeasible, feasible non-best, then feasible best. assert len(figure.collections) == 3 data = [(1, 0, 1, 1), (1, 2, 2, 0)] else: # The enqueue order of trial is: infeasible, feasible. assert len(figure.collections) == 2 data = [(1, 0, 1), (1, 2, 0)] else: if include_dominated_trials: # The last elements come from dominated trial that is enqueued firstly. assert len(figure.collections) == 2 data = [(0, 1, 1, 1), (2, 0, 2, 1)] else: assert len(figure.collections) == 1 data = [(0, 1), (2, 0)] _check_data(figure, "x", data[actual_axis_order[0]]) _check_data(figure, "y", data[actual_axis_order[1]]) plt.savefig(BytesIO()) # Test with `target_names` argument. error_message = "The length of `target_names` is supposed to be 2." with pytest.raises(ValueError, match=error_message): plot_pareto_front( study=study, target_names=[], include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, targets=targets, ) with pytest.raises(ValueError, match=error_message): plot_pareto_front( study=study, target_names=["Foo"], include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, targets=targets, ) with pytest.raises(ValueError, match=error_message): plot_pareto_front( study=study, target_names=["Foo", "Bar", "Baz"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, constraints_func=constraints_func, targets=targets, ) target_names = ["Foo", "Bar"] figure = plot_pareto_front( study=study, target_names=target_names, include_dominated_trials=include_dominated_trials, axis_order=axis_order, constraints_func=constraints_func, targets=targets, ) assert len(figure.get_lines()) == 0 if axis_order is None: assert figure.get_xlabel() == target_names[0] assert figure.get_ylabel() == target_names[1] else: assert figure.get_xlabel() == target_names[axis_order[0]] assert figure.get_ylabel() == target_names[axis_order[1]]
def test_plot_pareto_front_3d(include_dominated_trials: bool, axis_order: Optional[List[int]]) -> None: # Test with no trial. study = optuna.create_study( directions=["minimize", "minimize", "minimize"]) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert not figure.has_data() # Test with three trials. study.enqueue_trial({"x": 1, "y": 1, "z": 1}) study.enqueue_trial({"x": 1, "y": 0, "z": 1}) study.enqueue_trial({"x": 1, "y": 1, "z": 0}) study.optimize( lambda t: [ t.suggest_int("x", 0, 1), t.suggest_int("y", 0, 1), t.suggest_int("z", 0, 1) ], n_trials=3, ) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert figure.has_data() # Test with `target_names` argument. with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=[], include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo", "Bar"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo", "Bar", "Baz", "Qux"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) target_names = ["Foo", "Bar", "Baz"] figure = plot_pareto_front(study=study, target_names=target_names, axis_order=axis_order) assert figure.has_data()
def test_plot_pareto_front_2d( include_dominated_trials: bool, axis_order: Optional[List[int]], targets: Optional[Callable[[FrozenTrial], Sequence[float]]], ) -> None: if axis_order is not None and targets is not None: pytest.skip("skip using both axis_order and targets") # Test with no trial. study = optuna.create_study(directions=["minimize", "minimize"]) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.get_lines()) == 0 plt.savefig(BytesIO()) # Test with three trials. study.enqueue_trial({"x": 1, "y": 1}) study.enqueue_trial({"x": 1, "y": 0}) study.enqueue_trial({"x": 0, "y": 1}) study.optimize( lambda t: [t.suggest_int("x", 0, 1), t.suggest_int("y", 0, 1)], n_trials=3) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, targets=targets, ) assert len(figure.get_lines()) == 0 if axis_order is not None: pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]])[:, axis_order] else: pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]]) assert pareto_front_points.shape == (2, 2) path_offsets = list( map(lambda pc: pc.get_offsets(), figure.findobj(PathCollection))) exists_pareto_front = any( map(lambda po: allclose_as_set(po, pareto_front_points), path_offsets)) exists_dominated_trials = any( map(lambda po: allclose_as_set(po, np.array([[1.0, 1.0]])), path_offsets)) assert exists_pareto_front if include_dominated_trials: assert exists_dominated_trials plt.savefig(BytesIO()) # Test with `target_names` argument. with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=[], include_dominated_trials=include_dominated_trials, targets=targets, ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo"], include_dominated_trials=include_dominated_trials, targets=targets, ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo", "Bar", "Baz"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, targets=targets, ) target_names = ["Foo", "Bar"] figure = plot_pareto_front( study=study, target_names=target_names, include_dominated_trials=include_dominated_trials, axis_order=axis_order, targets=targets, ) assert len(figure.get_lines()) == 0 if axis_order is None: assert figure.get_xlabel() == target_names[0] assert figure.get_ylabel() == target_names[1] else: assert figure.get_xlabel() == target_names[axis_order[0]] assert figure.get_ylabel() == target_names[axis_order[1]] if axis_order is not None: pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]])[:, axis_order] else: pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]]) assert pareto_front_points.shape == (2, 2) path_offsets = list( map(lambda pc: pc.get_offsets(), figure.findobj(PathCollection))) exists_pareto_front = any( map(lambda po: allclose_as_set(po, pareto_front_points), path_offsets)) exists_dominated_trials = any( map(lambda po: allclose_as_set(po, np.array([[1.0, 1.0]])), path_offsets)) assert exists_pareto_front if include_dominated_trials: assert exists_dominated_trials plt.savefig(BytesIO())