def test_plot_pareto_front_unsupported_dimensions( include_dominated_trials: bool) -> None: # Unsupported: n_objectives == 1. with pytest.raises(ValueError): study = optuna.create_study(directions=["minimize"]) study.optimize(lambda t: [0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials) with pytest.raises(ValueError): study = optuna.create_study(direction="minimize") study.optimize(lambda t: [0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials) # Supported: n_objectives == 2. study = optuna.create_study(directions=["minimize", "minimize"]) study.optimize(lambda t: [0, 0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials) # Supported: n_objectives == 3. study = optuna.create_study( directions=["minimize", "minimize", "minimize"]) study.optimize(lambda t: [0, 0, 0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials) # Unsupported: n_objectives == 4. with pytest.raises(ValueError): study = optuna.create_study( directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda t: [0, 0, 0, 0], n_trials=1) plot_pareto_front(study=study, include_dominated_trials=include_dominated_trials)
def test_plot_pareto_front_using_axis_order_and_targets() -> None: study = optuna.create_study( directions=["minimize", "minimize", "minimize"]) with pytest.raises( ValueError, match="Using both `targets` and `axis_order` is not supported." " Use either `targets` or `axis_order`.", ): plot_pareto_front( study=study, axis_order=[0, 1, 2], targets=lambda t: (t.values[0], t.values[1], t.values[2]), )
def test_plot_pareto_front_targets_without_target_names() -> None: study = optuna.create_study( directions=["minimize", "minimize", "minimize"]) with pytest.raises( ValueError, match= "If `targets` is specified for empty studies, `target_names` must be specified.", ): plot_pareto_front( study=study, target_names=None, targets=lambda t: (t.values[0], t.values[1], t.values[2]), )
def test_plot_pareto_front_invalid_target_values() -> None: study = optuna.create_study( directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda _: [0, 0, 0, 0], n_trials=3) with pytest.raises( ValueError, match= "targets` should return a sequence of target values. your `targets`" " returns <class 'float'>", ): plot_pareto_front( study=study, targets=lambda t: t.values[0], )
def test_plot_pareto_front_n_targets_unsupported( targets: Callable[[FrozenTrial], Sequence[float]]) -> None: study = optuna.create_study( directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda _: [0, 0, 0, 0], n_trials=3) n_targets = len(targets(study.best_trials[0])) with pytest.raises( ValueError, match="`plot_pareto_front` function only supports 2 or 3 targets." " you used {} targets now.".format(n_targets), ): plot_pareto_front( study=study, targets=targets, )
def test_color_map(direction: str) -> None: study = prepare_study_with_trials(with_c_d=False, direction=direction, n_objectives=2) # Since `plot_pareto_front`'s colormap depends on only trial.number, # `reversecale` is not in the plot. marker = plot_pareto_front(study).data[0]["marker"] assert COLOR_SCALE == [v[1] for v in marker["colorscale"]] assert "reversecale" not in marker
def test_plot_pareto_front_invalid_axis_order( dimension: int, include_dominated_trials: bool, use_constraints_func: bool) -> None: study = optuna.create_study(directions=["minimize"] * dimension) constraints_func = (lambda x: [-1.0]) if use_constraints_func else None # Invalid: len(axis_order) != dimension with pytest.raises(ValueError): invalid_axis_order = list(range(dimension + 1)) assert len(invalid_axis_order) != dimension plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, ) # Invalid: np.unique(axis_order).size != dimension with pytest.raises(ValueError): invalid_axis_order = list(range(dimension)) invalid_axis_order[1] = invalid_axis_order[0] assert np.unique(invalid_axis_order).size != dimension plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, ) # Invalid: max(axis_order) > (dimension - 1) with pytest.raises(ValueError): invalid_axis_order = list(range(dimension)) invalid_axis_order[-1] += 1 assert max(invalid_axis_order) > (dimension - 1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, ) # Invalid: min(axis_order) < 0 with pytest.raises(ValueError): study = optuna.create_study(directions=["minimize", "minimize"]) invalid_axis_order = list(range(dimension)) invalid_axis_order[0] -= 1 assert min(invalid_axis_order) < 0 plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=invalid_axis_order, constraints_func=constraints_func, )
def test_plot_pareto_front_unsupported_dimensions( include_dominated_trials: bool, use_constraints_func: bool) -> None: constraints_func = (lambda x: [-1.0]) if use_constraints_func else None # Unsupported: n_objectives == 1. with pytest.raises(ValueError): study = optuna.create_study(directions=["minimize"]) study.optimize(lambda t: [0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, ) with pytest.raises(ValueError): study = optuna.create_study(direction="minimize") study.optimize(lambda t: [0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, ) # Unsupported: n_objectives == 4. with pytest.raises(ValueError): study = optuna.create_study( directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda t: [0, 0, 0, 0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, )
def test_color_map(direction: str) -> None: study = create_study(directions=[direction, direction]) for i in range(3): study.add_trial( create_trial( values=[float(i), float(i)], params={"param_a": 1.0, "param_b": 2.0}, distributions={ "param_a": FloatDistribution(0.0, 3.0), "param_b": FloatDistribution(0.0, 3.0), }, ) ) # Since `plot_pareto_front`'s colormap depends on only trial.number, # `reversecale` is not in the plot. marker = plot_pareto_front(study).data[0]["marker"] assert COLOR_SCALE == [v[1] for v in marker["colorscale"]] assert "reversecale" not in marker
def test_plot_pareto_front_unsupported_dimensions( include_dominated_trials: bool, use_constraints_func: bool) -> None: constraints_func = (lambda _: [-1.0]) if use_constraints_func else None error_message = ( "`plot_pareto_front` function only supports 2 or 3 objective" " studies when using `targets` is `None`. Please use `targets`" " if your objective studies have more than 3 objectives.") # Unsupported: n_objectives == 1. with pytest.raises(ValueError, match=error_message): study = optuna.create_study(directions=["minimize"]) study.optimize(lambda _: [0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, ) with pytest.raises(ValueError, match=error_message): study = optuna.create_study(direction="minimize") study.optimize(lambda _: [0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, ) # Unsupported: n_objectives == 4. with pytest.raises(ValueError, match=error_message): study = optuna.create_study( directions=["minimize", "minimize", "minimize", "minimize"]) study.optimize(lambda _: [0, 0, 0, 0], n_trials=1) plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, constraints_func=constraints_func, )
def test_plot_pareto_front_2d( include_dominated_trials: bool, use_constraints_func: bool, axis_order: Optional[List[int]], targets: Optional[Callable[[FrozenTrial], Sequence[float]]], ) -> None: if axis_order is not None and targets is not None: pytest.skip("skip using both axis_order and targets") # Test with no trial. study = optuna.create_study(directions=["minimize", "minimize"]) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.data) == 2 assert (figure.data[1]["x"] + figure.data[0]["x"]) == () assert (figure.data[1]["y"] + figure.data[0]["y"]) == () # Test with four trials. study.enqueue_trial({"x": 1, "y": 2}) study.enqueue_trial({"x": 1, "y": 1}) study.enqueue_trial({"x": 0, "y": 2}) study.enqueue_trial({"x": 1, "y": 0}) study.optimize( lambda t: [t.suggest_int("x", 0, 2), t.suggest_int("y", 0, 2)], n_trials=4) constraints_func: Optional[Callable[[FrozenTrial], Sequence[float]]] if use_constraints_func: # (x, y) = (1, 0) is infeasible; others are feasible. def constraints_func(t: FrozenTrial) -> Sequence[float]: return [1.0 ] if t.params["x"] == 1 and t.params["y"] == 0 else [-1.0] else: constraints_func = None figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, constraints_func=constraints_func, targets=targets, ) actual_axis_order = axis_order or [0, 1] if use_constraints_func: assert len(figure.data) == 3 if include_dominated_trials: # The enqueue order of trial is: infeasible, feasible non-best, then feasible best. data = [(1, 0, 1, 1), (1, 2, 2, 0)] # type: ignore else: # The enqueue order of trial is: infeasible, feasible. data = [(1, 0, 1), (1, 2, 0)] # type: ignore else: assert len(figure.data) == 2 if include_dominated_trials: # The last elements come from dominated trial that is enqueued firstly. data = [(0, 1, 1, 1), (2, 0, 2, 1)] # type: ignore else: data = [(0, 1), (2, 0)] # type: ignore _check_data(figure, "x", data[actual_axis_order[0]]) _check_data(figure, "y", data[actual_axis_order[1]]) titles = ["Objective {}".format(i) for i in range(2)] assert figure.layout.xaxis.title.text == titles[actual_axis_order[0]] assert figure.layout.yaxis.title.text == titles[actual_axis_order[1]] # Test with `target_names` argument. with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=[], include_dominated_trials=include_dominated_trials, targets=targets, ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo"], include_dominated_trials=include_dominated_trials, targets=targets, ) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo", "Bar", "Baz"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, targets=targets, ) target_names = ["Foo", "Bar"] figure = plot_pareto_front( study=study, target_names=target_names, include_dominated_trials=include_dominated_trials, axis_order=axis_order, constraints_func=constraints_func, targets=targets, ) assert figure.layout.xaxis.title.text == target_names[actual_axis_order[0]] assert figure.layout.yaxis.title.text == target_names[actual_axis_order[1]]
def test_plot_pareto_front_2d(include_dominated_trials: bool, axis_order: Optional[List[int]]) -> None: # Test with no trial. study = optuna.create_study(directions=["minimize", "minimize"]) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.data) == 2 assert (figure.data[1]["x"] + figure.data[0]["x"]) == () assert (figure.data[1]["y"] + figure.data[0]["y"]) == () # Test with three trials. study.enqueue_trial({"x": 1, "y": 1}) study.enqueue_trial({"x": 1, "y": 0}) study.enqueue_trial({"x": 0, "y": 1}) study.optimize( lambda t: [t.suggest_int("x", 0, 1), t.suggest_int("y", 0, 1)], n_trials=3) figure = plot_pareto_front( study=study, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) assert len(figure.data) == 2 if include_dominated_trials: # The last elements come from dominated trial that is enqueued firstly. data = [(1, 0, 1), (0, 1, 1)] # type: ignore if axis_order is None: assert (figure.data[1]["x"] + figure.data[0]["x"]) == data[0] assert (figure.data[1]["y"] + figure.data[0]["y"]) == data[1] else: assert (figure.data[1]["x"] + figure.data[0]["x"]) == data[axis_order[0]] assert (figure.data[1]["y"] + figure.data[0]["y"]) == data[axis_order[1]] else: data = [(1, 0), (0, 1)] # type: ignore if axis_order is None: assert (figure.data[1]["x"] + figure.data[0]["x"]) == data[0] assert (figure.data[1]["y"] + figure.data[0]["y"]) == data[1] else: assert (figure.data[1]["x"] + figure.data[0]["x"]) == data[axis_order[0]] assert (figure.data[1]["y"] + figure.data[0]["y"]) == data[axis_order[1]] titles = ["Objective {}".format(i) for i in range(2)] if axis_order is None: assert figure.layout.xaxis.title.text == titles[0] assert figure.layout.yaxis.title.text == titles[1] else: assert figure.layout.xaxis.title.text == titles[axis_order[0]] assert figure.layout.yaxis.title.text == titles[axis_order[1]] # Test with `target_names` argument. with pytest.raises(ValueError): plot_pareto_front(study=study, target_names=[], include_dominated_trials=include_dominated_trials) with pytest.raises(ValueError): plot_pareto_front(study=study, target_names=["Foo"], include_dominated_trials=include_dominated_trials) with pytest.raises(ValueError): plot_pareto_front( study=study, target_names=["Foo", "Bar", "Baz"], include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) target_names = ["Foo", "Bar"] figure = plot_pareto_front( study=study, target_names=target_names, include_dominated_trials=include_dominated_trials, axis_order=axis_order, ) if axis_order is None: assert figure.layout.xaxis.title.text == target_names[0] assert figure.layout.yaxis.title.text == target_names[1] else: assert figure.layout.xaxis.title.text == target_names[axis_order[0]] assert figure.layout.yaxis.title.text == target_names[axis_order[1]]