def test_constraints_func_experimental_warning() -> None:
study = optuna.create_study(directions=["minimize", "minimize"])
def constraints_func(t: FrozenTrial) -> Sequence[float]:
return [1.0]
with pytest.warns(optuna.exceptions.ExperimentalWarning):
plot_pareto_front(
study=study,
constraints_func=constraints_func,
)
def test_plot_pareto_front_targets_without_target_names() -> None:
study = optuna.create_study(directions=["minimize", "minimize", "minimize"])
with pytest.raises(
ValueError,
match="If `targets` is specified for empty studies, `target_names` must be specified.",
):
plot_pareto_front(
study=study,
target_names=None,
targets=lambda t: (t.values[0], t.values[1], t.values[2]),
)
def test_plot_pareto_front_2d(
include_dominated_trials: bool, axis_order: Optional[List[int]]
) -> None:
# Test with no trial.
study = optuna.create_study(directions=["minimize", "minimize"])
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.get_lines()) == 0
# Test with three trials.
study.enqueue_trial({"x": 1, "y": 1})
study.enqueue_trial({"x": 1, "y": 0})
study.enqueue_trial({"x": 0, "y": 1})
study.optimize(lambda t: [t.suggest_int("x", 0, 1), t.suggest_int("y", 0, 1)], n_trials=3)
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.get_lines()) == 0
# Test with `target_names` argument.
with pytest.raises(ValueError):
plot_pareto_front(
study=study, target_names=[], include_dominated_trials=include_dominated_trials
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study, target_names=["Foo"], include_dominated_trials=include_dominated_trials
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo", "Bar", "Baz"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
target_names = ["Foo", "Bar"]
figure = plot_pareto_front(
study=study,
target_names=target_names,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.get_lines()) == 0
if axis_order is None:
assert figure.get_xlabel() == target_names[0]
assert figure.get_ylabel() == target_names[1]
else:
assert figure.get_xlabel() == target_names[axis_order[0]]
assert figure.get_ylabel() == target_names[axis_order[1]]
def test_plot_pareto_front_using_axis_order_and_targets() -> None:
study = optuna.create_study(directions=["minimize", "minimize", "minimize"])
with pytest.raises(
ValueError,
match="Using both `targets` and `axis_order` is not supported."
" Use either `targets` or `axis_order`.",
):
plot_pareto_front(
study=study,
axis_order=[0, 1, 2],
targets=lambda t: (t.values[0], t.values[1], t.values[2]),
)
def test_plot_pareto_front_invalid_target_values() -> None:
study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda _: [0, 0, 0, 0], n_trials=3)
with pytest.raises(
ValueError,
match="targets` should return a sequence of target values. your `targets`"
" returns <class 'float'>",
):
plot_pareto_front(
study=study,
targets=lambda t: t.values[0],
)
def test_plot_pareto_front_n_targets_unsupported(
targets: Callable[[FrozenTrial], Sequence[float]]
) -> None:
study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda _: [0, 0, 0, 0], n_trials=3)
n_targets = len(targets(study.best_trials[0]))
with pytest.raises(
ValueError,
match="`plot_pareto_front` function only supports 2 or 3 targets."
" you used {} targets now.".format(n_targets),
):
plot_pareto_front(
study=study,
targets=targets,
)
def test_plot_pareto_front_invalid_target_values(
targets: Optional[Callable[[FrozenTrial], Sequence[float]]]) -> None:
study = optuna.create_study(
directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda t: [0, 0, 0, 0], n_trials=3)
with pytest.raises(
ValueError,
match=
"targets` should return a sequence of target values. your `targets`"
" returns <class 'float'>",
):
plot_pareto_front(
study=study,
targets=targets,
)
def test_plot_pareto_front_invalid_axis_order(
dimension: int, include_dominated_trials: bool, use_constraints_func: bool
) -> None:
study = optuna.create_study(directions=["minimize"] * dimension)
study.optimize(lambda _: [0] * dimension, n_trials=1)
constraints_func = (lambda _: [-1.0]) if use_constraints_func else None
# Invalid: len(axis_order) != dimension
with pytest.raises(ValueError):
invalid_axis_order = list(range(dimension + 1))
assert len(invalid_axis_order) != dimension
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
# Invalid: np.unique(axis_order).size != dimension
with pytest.raises(ValueError):
invalid_axis_order = list(range(dimension))
invalid_axis_order[1] = invalid_axis_order[0]
assert np.unique(invalid_axis_order).size != dimension
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
# Invalid: max(axis_order) > (dimension - 1)
with pytest.raises(ValueError):
invalid_axis_order = list(range(dimension))
invalid_axis_order[-1] += 1
assert max(invalid_axis_order) > (dimension - 1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
# Invalid: min(axis_order) < 0
with pytest.raises(ValueError):
study = optuna.create_study(directions=["minimize", "minimize"])
study.optimize(lambda _: [0] * 2, n_trials=1)
invalid_axis_order = list(range(dimension))
invalid_axis_order[0] -= 1
assert min(invalid_axis_order) < 0
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=invalid_axis_order,
constraints_func=constraints_func,
)
def _log_plots(run,
study: optuna.Study,
visualization_backend='plotly',
log_plot_contour=True,
log_plot_edf=True,
log_plot_parallel_coordinate=True,
log_plot_param_importances=True,
log_plot_pareto_front=True,
log_plot_slice=True,
log_plot_intermediate_values=True,
log_plot_optimization_history=True,
):
if visualization_backend == 'matplotlib':
import optuna.visualization.matplotlib as vis
elif visualization_backend == 'plotly':
import optuna.visualization as vis
else:
raise NotImplementedError(f'{visualization_backend} visualisation backend is not implemented')
if vis.is_available:
params = list(p_name for t in study.trials for p_name in t.params.keys())
if log_plot_contour and any(params):
run['visualizations/plot_contour'] = neptune.types.File.as_html(vis.plot_contour(study))
if log_plot_edf:
run['visualizations/plot_edf'] = neptune.types.File.as_html(vis.plot_edf(study))
if log_plot_parallel_coordinate:
run['visualizations/plot_parallel_coordinate'] = \
neptune.types.File.as_html(vis.plot_parallel_coordinate(study))
if log_plot_param_importances and len(study.get_trials(states=(optuna.trial.TrialState.COMPLETE, optuna.trial.TrialState.PRUNED,))) > 1:
try:
run['visualizations/plot_param_importances'] = neptune.types.File.as_html(vis.plot_param_importances(study))
except (RuntimeError, ValueError, ZeroDivisionError):
# Unable to compute importances
pass
if log_plot_pareto_front and study._is_multi_objective() and visualization_backend == 'plotly':
run['visualizations/plot_pareto_front'] = neptune.types.File.as_html(vis.plot_pareto_front(study))
if log_plot_slice and any(params):
run['visualizations/plot_slice'] = neptune.types.File.as_html(vis.plot_slice(study))
if log_plot_intermediate_values and any(trial.intermediate_values for trial in study.trials):
# Intermediate values plot if available only if the above condition is met
run['visualizations/plot_intermediate_values'] = \
neptune.types.File.as_html(vis.plot_intermediate_values(study))
if log_plot_optimization_history:
run['visualizations/plot_optimization_history'] = \
neptune.types.File.as_html(vis.plot_optimization_history(study))
def test_plot_pareto_front_dimensions(include_dominated_trials: bool) -> None:
# Unsupported: n_objectives == 1.
with pytest.raises(ValueError):
study = optuna.create_study(directions=["minimize"])
study.optimize(lambda t: [0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
with pytest.raises(ValueError):
study = optuna.create_study(direction="minimize")
study.optimize(lambda t: [0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
# Supported: n_objectives == 2.
study = optuna.create_study(directions=["minimize", "minimize"])
study.optimize(lambda t: [0, 0], n_trials=1)
figure = plot_pareto_front(
study=study, include_dominated_trials=include_dominated_trials)
assert figure.has_data()
# Supported: n_objectives == 3.
study = optuna.create_study(
directions=["minimize", "minimize", "minimize"])
study.optimize(lambda t: [0, 0, 0], n_trials=1)
figure = plot_pareto_front(
study=study, include_dominated_trials=include_dominated_trials)
assert figure.has_data()
# Unsupported: n_objectives == 4.
with pytest.raises(ValueError):
study = optuna.create_study(
directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda t: [0, 0, 0, 0], n_trials=1)
plot_pareto_front(study=study,
include_dominated_trials=include_dominated_trials)
def test_plot_pareto_front_unsupported_dimensions(
include_dominated_trials: bool, use_constraints_func: bool
) -> None:
constraints_func = (lambda _: [-1.0]) if use_constraints_func else None
error_message = (
"`plot_pareto_front` function only supports 2 or 3 objective"
" studies when using `targets` is `None`. Please use `targets`"
" if your objective studies have more than 3 objectives."
)
# Unsupported: n_objectives == 1.
with pytest.raises(ValueError, match=error_message):
study = optuna.create_study(directions=["minimize"])
study.optimize(lambda _: [0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
with pytest.raises(ValueError, match=error_message):
study = optuna.create_study(direction="minimize")
study.optimize(lambda _: [0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
# Unsupported: n_objectives == 4.
with pytest.raises(ValueError, match=error_message):
study = optuna.create_study(directions=["minimize", "minimize", "minimize", "minimize"])
study.optimize(lambda _: [0, 0, 0, 0], n_trials=1)
plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
)
def test_plot_pareto_front_2d(
include_dominated_trials: bool,
axis_order: Optional[List[int]],
use_constraints_func: bool,
targets: Optional[Callable[[FrozenTrial], Sequence[float]]],
) -> None:
if axis_order is not None and targets is not None:
pytest.skip("skip using both axis_order and targets")
# Test with no trial.
study = optuna.create_study(directions=["minimize", "minimize"])
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.get_lines()) == 0
plt.savefig(BytesIO())
# Test with four trials.
study.enqueue_trial({"x": 1, "y": 2})
study.enqueue_trial({"x": 1, "y": 1})
study.enqueue_trial({"x": 0, "y": 2})
study.enqueue_trial({"x": 1, "y": 0})
study.optimize(lambda t: [t.suggest_int("x", 0, 2), t.suggest_int("y", 0, 2)], n_trials=4)
constraints_func: Optional[Callable[[FrozenTrial], Sequence[float]]]
if use_constraints_func:
# (x, y) = (1, 0) is infeasible; others are feasible.
def constraints_func(t: FrozenTrial) -> Sequence[float]:
return [1.0] if t.params["x"] == 1 and t.params["y"] == 0 else [-1.0]
else:
constraints_func = None
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
constraints_func=constraints_func,
targets=targets,
)
assert len(figure.get_lines()) == 0
actual_axis_order = axis_order or [0, 1]
data: List[Tuple[int, ...]]
if use_constraints_func:
if include_dominated_trials:
# The enqueue order of trial is: infeasible, feasible non-best, then feasible best.
assert len(figure.collections) == 3
data = [(1, 0, 1, 1), (1, 2, 2, 0)]
else:
# The enqueue order of trial is: infeasible, feasible.
assert len(figure.collections) == 2
data = [(1, 0, 1), (1, 2, 0)]
else:
if include_dominated_trials:
# The last elements come from dominated trial that is enqueued firstly.
assert len(figure.collections) == 2
data = [(0, 1, 1, 1), (2, 0, 2, 1)]
else:
assert len(figure.collections) == 1
data = [(0, 1), (2, 0)]
_check_data(figure, "x", data[actual_axis_order[0]])
_check_data(figure, "y", data[actual_axis_order[1]])
plt.savefig(BytesIO())
# Test with `target_names` argument.
error_message = "The length of `target_names` is supposed to be 2."
with pytest.raises(ValueError, match=error_message):
plot_pareto_front(
study=study,
target_names=[],
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
targets=targets,
)
with pytest.raises(ValueError, match=error_message):
plot_pareto_front(
study=study,
target_names=["Foo"],
include_dominated_trials=include_dominated_trials,
constraints_func=constraints_func,
targets=targets,
)
with pytest.raises(ValueError, match=error_message):
plot_pareto_front(
study=study,
target_names=["Foo", "Bar", "Baz"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
constraints_func=constraints_func,
targets=targets,
)
target_names = ["Foo", "Bar"]
figure = plot_pareto_front(
study=study,
target_names=target_names,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
constraints_func=constraints_func,
targets=targets,
)
assert len(figure.get_lines()) == 0
if axis_order is None:
assert figure.get_xlabel() == target_names[0]
assert figure.get_ylabel() == target_names[1]
else:
assert figure.get_xlabel() == target_names[axis_order[0]]
assert figure.get_ylabel() == target_names[axis_order[1]]
def test_plot_pareto_front_3d(include_dominated_trials: bool,
axis_order: Optional[List[int]]) -> None:
# Test with no trial.
study = optuna.create_study(
directions=["minimize", "minimize", "minimize"])
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert not figure.has_data()
# Test with three trials.
study.enqueue_trial({"x": 1, "y": 1, "z": 1})
study.enqueue_trial({"x": 1, "y": 0, "z": 1})
study.enqueue_trial({"x": 1, "y": 1, "z": 0})
study.optimize(
lambda t: [
t.suggest_int("x", 0, 1),
t.suggest_int("y", 0, 1),
t.suggest_int("z", 0, 1)
],
n_trials=3,
)
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert figure.has_data()
# Test with `target_names` argument.
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=[],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo", "Bar"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo", "Bar", "Baz", "Qux"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
target_names = ["Foo", "Bar", "Baz"]
figure = plot_pareto_front(study=study,
target_names=target_names,
axis_order=axis_order)
assert figure.has_data()
def test_plot_pareto_front_2d(
include_dominated_trials: bool,
axis_order: Optional[List[int]],
targets: Optional[Callable[[FrozenTrial], Sequence[float]]],
) -> None:
if axis_order is not None and targets is not None:
pytest.skip("skip using both axis_order and targets")
# Test with no trial.
study = optuna.create_study(directions=["minimize", "minimize"])
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
)
assert len(figure.get_lines()) == 0
plt.savefig(BytesIO())
# Test with three trials.
study.enqueue_trial({"x": 1, "y": 1})
study.enqueue_trial({"x": 1, "y": 0})
study.enqueue_trial({"x": 0, "y": 1})
study.optimize(
lambda t: [t.suggest_int("x", 0, 1),
t.suggest_int("y", 0, 1)],
n_trials=3)
figure = plot_pareto_front(
study=study,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
targets=targets,
)
assert len(figure.get_lines()) == 0
if axis_order is not None:
pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]])[:, axis_order]
else:
pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]])
assert pareto_front_points.shape == (2, 2)
path_offsets = list(
map(lambda pc: pc.get_offsets(), figure.findobj(PathCollection)))
exists_pareto_front = any(
map(lambda po: allclose_as_set(po, pareto_front_points), path_offsets))
exists_dominated_trials = any(
map(lambda po: allclose_as_set(po, np.array([[1.0, 1.0]])),
path_offsets))
assert exists_pareto_front
if include_dominated_trials:
assert exists_dominated_trials
plt.savefig(BytesIO())
# Test with `target_names` argument.
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=[],
include_dominated_trials=include_dominated_trials,
targets=targets,
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo"],
include_dominated_trials=include_dominated_trials,
targets=targets,
)
with pytest.raises(ValueError):
plot_pareto_front(
study=study,
target_names=["Foo", "Bar", "Baz"],
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
targets=targets,
)
target_names = ["Foo", "Bar"]
figure = plot_pareto_front(
study=study,
target_names=target_names,
include_dominated_trials=include_dominated_trials,
axis_order=axis_order,
targets=targets,
)
assert len(figure.get_lines()) == 0
if axis_order is None:
assert figure.get_xlabel() == target_names[0]
assert figure.get_ylabel() == target_names[1]
else:
assert figure.get_xlabel() == target_names[axis_order[0]]
assert figure.get_ylabel() == target_names[axis_order[1]]
if axis_order is not None:
pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]])[:, axis_order]
else:
pareto_front_points = np.array([[1.0, 0.0], [0.0, 1.0]])
assert pareto_front_points.shape == (2, 2)
path_offsets = list(
map(lambda pc: pc.get_offsets(), figure.findobj(PathCollection)))
exists_pareto_front = any(
map(lambda po: allclose_as_set(po, pareto_front_points), path_offsets))
exists_dominated_trials = any(
map(lambda po: allclose_as_set(po, np.array([[1.0, 1.0]])),
path_offsets))
assert exists_pareto_front
if include_dominated_trials:
assert exists_dominated_trials
plt.savefig(BytesIO())
