def test_plot_pareto_front_unsupported_dimensions(
include_dominated_trials: bool) -> None:
# Unsupported: n_objectives == 1.
with pytest.raises(ValueError):
study = optuna.create_study(directions=["minimize"])
study.optimize(lambda t: [0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
with pytest.raises(ValueError):
study = optuna.create_study(direction="minimize")
study.optimize(lambda t: [0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
# Supported: n_objectives == 2.
study = optuna.create_study(directions=["minimize", "minimize"])
study.optimize(lambda t: [0, 0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
# Supported: n_objectives == 3.
study = optuna.create_study(
directions=["minimize", "minimize", "minimize"])
study.optimize(lambda t: [0, 0, 0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
# Unsupported: n_objectives == 4.
with pytest.raises(ValueError):
study = optuna.create_study(
directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda t: [0, 0, 0, 0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
def test_plot_pareto_front_using_axis_order_and_targets() -> None:
study = optuna.create_study(
directions=["minimize", "minimize", "minimize"])
with pytest.raises(
ValueError,
match="Using both `targets` and `axis_order` is not supported."
" Use either `targets` or `axis_order`.",
):
plot_pareto_front(
study=study,
axis_order=[0, 1, 2],
targets=lambda t: (t.values[0], t.values[1], t.values[2]),
)
def test_plot_pareto_front_targets_without_target_names() -> None:
study = optuna.create_study(
directions=["minimize", "minimize", "minimize"])
with pytest.raises(
ValueError,
match=
"If `targets` is specified for empty studies, `target_names` must be specified.",
):
plot_pareto_front(
study=study,
target_names=None,
targets=lambda t: (t.values[0], t.values[1], t.values[2]),
)
def test_plot_pareto_front_invalid_target_values() -> None:
study = optuna.create_study(
directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda _: [0, 0, 0, 0], n_trials=3)
with pytest.raises(
ValueError,
match=
"targets` should return a sequence of target values. your `targets`"
" returns <class 'float'>",
):
plot_pareto_front(
study=study,
targets=lambda t: t.values[0],
)
def test_plot_pareto_front_n_targets_unsupported(
targets: Callable[[FrozenTrial], Sequence[float]]) -> None:
study = optuna.create_study(
directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda _: [0, 0, 0, 0], n_trials=3)
n_targets = len(targets(study.best_trials[0]))
with pytest.raises(
ValueError,
match="`plot_pareto_front` function only supports 2 or 3 targets."
" you used {} targets now.".format(n_targets),
):
plot_pareto_front(
study=study,
targets=targets,
)
def test_color_map(direction: str) -> None:
study = prepare_study_with_trials(with_c_d=False,
direction=direction,
n_objectives=2)
# Since `plot_pareto_front`'s colormap depends on only trial.number,
# `reversecale` is not in the plot.
marker = plot_pareto_front(study).data[0]["marker"]
assert COLOR_SCALE == [v[1] for v in marker["colorscale"]]
assert "reversecale" not in marker
def test_plot_pareto_front_invalid_axis_order(
dimension: int, include_dominated_trials: bool,
use_constraints_func: bool) -> None:
study = optuna.create_study(directions=["minimize"] * dimension)
constraints_func = (lambda x: [-1.0]) if use_constraints_func else None
# Invalid: len(axis_order) != dimension
with pytest.raises(ValueError):
invalid_axis_order = list(range(dimension + 1))
assert len(invalid_axis_order) != dimension
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
# Invalid: np.unique(axis_order).size != dimension
with pytest.raises(ValueError):
invalid_axis_order = list(range(dimension))
invalid_axis_order[1] = invalid_axis_order[0]
assert np.unique(invalid_axis_order).size != dimension
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
# Invalid: max(axis_order) > (dimension - 1)
with pytest.raises(ValueError):
invalid_axis_order = list(range(dimension))
invalid_axis_order[-1] += 1
assert max(invalid_axis_order) > (dimension - 1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
# Invalid: min(axis_order) < 0
with pytest.raises(ValueError):
study = optuna.create_study(directions=["minimize", "minimize"])
invalid_axis_order = list(range(dimension))
invalid_axis_order[0] -= 1
assert min(invalid_axis_order) < 0
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
def test_plot_pareto_front_unsupported_dimensions(
include_dominated_trials: bool, use_constraints_func: bool) -> None:
constraints_func = (lambda x: [-1.0]) if use_constraints_func else None
# Unsupported: n_objectives == 1.
with pytest.raises(ValueError):
study = optuna.create_study(directions=["minimize"])
study.optimize(lambda t: [0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
with pytest.raises(ValueError):
study = optuna.create_study(direction="minimize")
study.optimize(lambda t: [0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
# Unsupported: n_objectives == 4.
with pytest.raises(ValueError):
study = optuna.create_study(
directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda t: [0, 0, 0, 0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
def test_color_map(direction: str) -> None:
study = create_study(directions=[direction, direction])
for i in range(3):
study.add_trial(
create_trial(
values=[float(i), float(i)],
params={"param_a": 1.0, "param_b": 2.0},
distributions={
"param_a": FloatDistribution(0.0, 3.0),
"param_b": FloatDistribution(0.0, 3.0),
},
)
)
# Since `plot_pareto_front`'s colormap depends on only trial.number,
# `reversecale` is not in the plot.
marker = plot_pareto_front(study).data[0]["marker"]
assert COLOR_SCALE == [v[1] for v in marker["colorscale"]]
assert "reversecale" not in marker
def test_plot_pareto_front_unsupported_dimensions(
include_dominated_trials: bool, use_constraints_func: bool) -> None:
constraints_func = (lambda _: [-1.0]) if use_constraints_func else None
error_message = (
"`plot_pareto_front` function only supports 2 or 3 objective"
" studies when using `targets` is `None`. Please use `targets`"
" if your objective studies have more than 3 objectives.")
# Unsupported: n_objectives == 1.
with pytest.raises(ValueError, match=error_message):
study = optuna.create_study(directions=["minimize"])
study.optimize(lambda _: [0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
with pytest.raises(ValueError, match=error_message):
study = optuna.create_study(direction="minimize")
study.optimize(lambda _: [0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
# Unsupported: n_objectives == 4.
with pytest.raises(ValueError, match=error_message):
study = optuna.create_study(
directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda _: [0, 0, 0, 0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
def test_plot_pareto_front_2d(
include_dominated_trials: bool,
use_constraints_func: bool,
axis_order: Optional[List[int]],
targets: Optional[Callable[[FrozenTrial], Sequence[float]]],
) -> None:
if axis_order is not None and targets is not None:
pytest.skip("skip using both axis_order and targets")
# Test with no trial.
study = optuna.create_study(directions=["minimize", "minimize"])
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.data) == 2
assert (figure.data[1]["x"] + figure.data[0]["x"]) == ()
assert (figure.data[1]["y"] + figure.data[0]["y"]) == ()
# Test with four trials.
study.enqueue_trial({"x": 1, "y": 2})
study.enqueue_trial({"x": 1, "y": 1})
study.enqueue_trial({"x": 0, "y": 2})
study.enqueue_trial({"x": 1, "y": 0})
study.optimize(
lambda t: [t.suggest_int("x", 0, 2),
t.suggest_int("y", 0, 2)],
n_trials=4)
constraints_func: Optional[Callable[[FrozenTrial], Sequence[float]]]
if use_constraints_func:
# (x, y) = (1, 0) is infeasible; others are feasible.
def constraints_func(t: FrozenTrial) -> Sequence[float]:
return [1.0
] if t.params["x"] == 1 and t.params["y"] == 0 else [-1.0]
else:
constraints_func = None
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
constraints_func=constraints_func,
targets=targets,
)
actual_axis_order = axis_order or [0, 1]
if use_constraints_func:
assert len(figure.data) == 3
if include_dominated_trials:
# The enqueue order of trial is: infeasible, feasible non-best, then feasible best.
data = [(1, 0, 1, 1), (1, 2, 2, 0)] # type: ignore
else:
# The enqueue order of trial is: infeasible, feasible.
data = [(1, 0, 1), (1, 2, 0)] # type: ignore
else:
assert len(figure.data) == 2
if include_dominated_trials:
# The last elements come from dominated trial that is enqueued firstly.
data = [(0, 1, 1, 1), (2, 0, 2, 1)] # type: ignore
else:
data = [(0, 1), (2, 0)] # type: ignore
_check_data(figure, "x", data[actual_axis_order[0]])
_check_data(figure, "y", data[actual_axis_order[1]])
titles = ["Objective {}".format(i) for i in range(2)]
assert figure.layout.xaxis.title.text == titles[actual_axis_order[0]]
assert figure.layout.yaxis.title.text == titles[actual_axis_order[1]]
# Test with `target_names` argument.
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=[],
include_dominated_trials=include_dominated_trials,
targets=targets,
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo"],
include_dominated_trials=include_dominated_trials,
targets=targets,
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo", "Bar", "Baz"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
targets=targets,
)
target_names = ["Foo", "Bar"]
figure = plot_pareto_front(
study=study,
target_names=target_names,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
constraints_func=constraints_func,
targets=targets,
)
assert figure.layout.xaxis.title.text == target_names[actual_axis_order[0]]
assert figure.layout.yaxis.title.text == target_names[actual_axis_order[1]]
def test_plot_pareto_front_2d(include_dominated_trials: bool,
axis_order: Optional[List[int]]) -> None:
# Test with no trial.
study = optuna.create_study(directions=["minimize", "minimize"])
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.data) == 2
assert (figure.data[1]["x"] + figure.data[0]["x"]) == ()
assert (figure.data[1]["y"] + figure.data[0]["y"]) == ()
# Test with three trials.
study.enqueue_trial({"x": 1, "y": 1})
study.enqueue_trial({"x": 1, "y": 0})
study.enqueue_trial({"x": 0, "y": 1})
study.optimize(
lambda t: [t.suggest_int("x", 0, 1),
t.suggest_int("y", 0, 1)],
n_trials=3)
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.data) == 2
if include_dominated_trials:
# The last elements come from dominated trial that is enqueued firstly.
data = [(1, 0, 1), (0, 1, 1)] # type: ignore
if axis_order is None:
assert (figure.data[1]["x"] + figure.data[0]["x"]) == data[0]
assert (figure.data[1]["y"] + figure.data[0]["y"]) == data[1]
else:
assert (figure.data[1]["x"] +
figure.data[0]["x"]) == data[axis_order[0]]
assert (figure.data[1]["y"] +
figure.data[0]["y"]) == data[axis_order[1]]
else:
data = [(1, 0), (0, 1)] # type: ignore
if axis_order is None:
assert (figure.data[1]["x"] + figure.data[0]["x"]) == data[0]
assert (figure.data[1]["y"] + figure.data[0]["y"]) == data[1]
else:
assert (figure.data[1]["x"] +
figure.data[0]["x"]) == data[axis_order[0]]
assert (figure.data[1]["y"] +
figure.data[0]["y"]) == data[axis_order[1]]
titles = ["Objective {}".format(i) for i in range(2)]
if axis_order is None:
assert figure.layout.xaxis.title.text == titles[0]
assert figure.layout.yaxis.title.text == titles[1]
else:
assert figure.layout.xaxis.title.text == titles[axis_order[0]]
assert figure.layout.yaxis.title.text == titles[axis_order[1]]
# Test with `target_names` argument.
with pytest.raises(ValueError):
plot_pareto_front(study=study,
target_names=[],
include_dominated_trials=include_dominated_trials)
with pytest.raises(ValueError):
plot_pareto_front(study=study,
target_names=["Foo"],
include_dominated_trials=include_dominated_trials)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo", "Bar", "Baz"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
target_names = ["Foo", "Bar"]
figure = plot_pareto_front(
study=study,
target_names=target_names,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
if axis_order is None:
assert figure.layout.xaxis.title.text == target_names[0]
assert figure.layout.yaxis.title.text == target_names[1]
else:
assert figure.layout.xaxis.title.text == target_names[axis_order[0]]
assert figure.layout.yaxis.title.text == target_names[axis_order[1]]
