def test_plot_param_importances() -> None:
# Test with no trial.
study = create_study()
figure = plot_param_importances(study)
assert len(figure.get_lines()) == 0
study = prepare_study_with_trials(with_c_d=True)
# Test with a trial.
figure = plot_param_importances(study)
assert len(figure.get_lines()) == 0
assert figure.get_xaxis().label.get_text(
) == "Importance for Objective Value"
# Test with an evaluator.
plot_param_importances(study,
evaluator=MeanDecreaseImpurityImportanceEvaluator())
assert len(figure.get_lines()) == 0
assert figure.get_xaxis().label.get_text(
) == "Importance for Objective Value"
# Test with a trial to select parameter.
figure = plot_param_importances(study, params=["param_b"])
assert len(figure.get_lines()) == 0
assert figure.get_xaxis().label.get_text(
) == "Importance for Objective Value"
# Test with a customized target value.
with pytest.warns(UserWarning):
figure = plot_param_importances(
study, target=lambda t: t.params["param_b"] + t.params["param_d"])
assert len(figure.get_lines()) == 0
# Test with a customized target name.
figure = plot_param_importances(study, target_name="Target Name")
assert len(figure.get_lines()) == 0
assert figure.get_xaxis().label.get_text() == "Importance for Target Name"
# Test with wrong parameters.
with pytest.raises(ValueError):
plot_param_importances(study, params=["optuna"])
# Ignore failed trials.
def fail_objective(_: Trial) -> float:
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError, ))
figure = plot_param_importances(study)
assert len(figure.get_lines()) == 0
def test_multi_objective_trial_with_infinite_value_ignored(
target_idx: int, inf_value: float, evaluator: BaseImportanceEvaluator,
n_trial: int) -> None:
def _multi_objective_function(trial: Trial) -> Tuple[float, float]:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 2, 4, log=True)
return x1, x2 * x3
seed = 13
study = create_study(directions=["minimize", "minimize"],
sampler=RandomSampler(seed=seed))
study.optimize(_multi_objective_function, n_trials=n_trial)
# A figure is created without a trial with an inf value.
plot_param_importances(study,
evaluator=evaluator,
target=lambda t: t.values[target_idx])
with BytesIO() as byte_io:
plt.savefig(byte_io)
figure_with_inf = byte_io.getvalue()
# A trial with an inf value is added into the study manually.
study.add_trial(
create_trial(
values=[inf_value, inf_value],
params={
"x1": 1.0,
"x2": 1.0,
"x3": 3.0
},
distributions={
"x1": FloatDistribution(low=0.1, high=3),
"x2": FloatDistribution(low=0.1, high=3, log=True),
"x3": FloatDistribution(low=2, high=4, log=True),
},
))
# A figure is created with a trial with an inf value.
plot_param_importances(study,
evaluator=evaluator,
target=lambda t: t.values[target_idx])
with BytesIO() as byte_io:
plt.savefig(byte_io)
figure_without_inf = byte_io.getvalue()
# Obtained figures should be the same between with inf and without inf,
# because the last trial whose objective value is an inf is ignored.
assert len(figure_without_inf) > 0
assert figure_without_inf == figure_with_inf
def test_plot_param_importances() -> None:
# Test with no trial.
study = create_study()
figure = plot_param_importances(study)
assert not figure.has_data()
study = prepare_study_with_trials(with_c_d=True)
# Test with a trial.
# TODO(ytknzw): Add more specific assertion with the test case.
figure = plot_param_importances(study)
assert figure.has_data()
# Test with an evaluator.
# TODO(ytknzw): Add more specific assertion with the test case.
plot_param_importances(study,
evaluator=MeanDecreaseImpurityImportanceEvaluator())
assert figure.has_data()
# Test with a trial to select parameter.
# TODO(ytknzw): Add more specific assertion with the test case.
figure = plot_param_importances(study, params=["param_b"])
assert figure.has_data()
# Test with a customized target value.
with pytest.warns(UserWarning):
figure = plot_param_importances(
study, target=lambda t: t.params["param_b"] + t.params["param_d"])
assert figure.has_data()
# Test with a customized target name.
figure = plot_param_importances(study, target_name="Target Name")
assert figure.has_data()
# Test with wrong parameters.
with pytest.raises(ValueError):
plot_param_importances(study, params=["optuna"])
# Ignore failed trials.
def fail_objective(_: Trial) -> float:
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError, ))
figure = plot_param_importances(study)
assert not figure.has_data()
def test_importance_scores_rendering() -> None:
study = prepare_study_with_trials()
ax = plot_param_importances(study)
# Test if importance scores are rendered.
text_objects = ax.figure.findobj(lambda obj: "Text" in str(obj))
importances = [patch.get_width() for patch in ax.patches]
labels = [
obj for obj in text_objects if obj.get_position()[0] in importances
]
assert len(labels) == 2
def test_switch_label_when_param_insignificant() -> None:
def _objective(trial: Trial) -> int:
x = trial.suggest_int("x", 0, 2)
_ = trial.suggest_int("y", -1, 1)
return x**2
study = create_study()
for x in range(1, 3):
study.enqueue_trial({"x": x, "y": 0})
study.optimize(_objective, n_trials=2)
ax = plot_param_importances(study)
# Test if label for `y` param has been switched to `<0.01`.
labels = ax.figure.findobj(lambda obj: "<0.01" in str(obj))
assert len(labels) == 1
def test_plot_param_importances() -> None:
# Test with no trial.
study = create_study()
figure = plot_param_importances(study)
assert len(figure.get_lines()) == 0
plt.savefig(BytesIO())
study = prepare_study_with_trials(with_c_d=True)
# Test with a trial.
figure = plot_param_importances(study)
bars = figure.findobj(
Rectangle)[:-1] # The last Rectangle is the plot itself.
plotted_data = [bar.get_width() for bar in bars]
# get_yticklabels returns a data structure of Text(0, 0, 'param_d').
labels = [label.get_text() for label in figure.get_yticklabels()]
assert len(figure.get_lines()) == 0
assert len(bars) == 2
assert set(labels) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in plotted_data), abs_tol=1e-5)
assert figure.xaxis.label.get_text() == "Importance for Objective Value"
plt.savefig(BytesIO())
# Test with an evaluator.
plot_param_importances(study,
evaluator=MeanDecreaseImpurityImportanceEvaluator())
bars = figure.findobj(
Rectangle)[:-1] # The last Rectangle is the plot itself.
plotted_data = [bar.get_width() for bar in bars]
labels = [label.get_text() for label in figure.get_yticklabels()]
assert len(figure.get_lines()) == 0
assert len(bars) == 2
assert set(labels) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in plotted_data), abs_tol=1e-5)
assert figure.xaxis.label.get_text() == "Importance for Objective Value"
plt.savefig(BytesIO())
# Test with a trial to select parameter.
figure = plot_param_importances(study, params=["param_b"])
bars = figure.findobj(
Rectangle)[:-1] # The last Rectangle is the plot itself.
plotted_data = [bar.get_width() for bar in bars]
labels = [label.get_text() for label in figure.get_yticklabels()]
assert len(figure.get_lines()) == 0
assert len(bars) == 1
assert set(labels) == set(("param_b", ))
assert math.isclose(1.0, sum(i for i in plotted_data), abs_tol=1e-5)
assert figure.xaxis.label.get_text() == "Importance for Objective Value"
plt.savefig(BytesIO())
# Test with a customized target value.
with pytest.warns(UserWarning):
figure = plot_param_importances(
study, target=lambda t: t.params["param_b"] + t.params["param_d"])
bars = figure.findobj(
Rectangle)[:-1] # The last Rectangle is the plot itself.
plotted_data = [bar.get_width() for bar in bars]
labels = [label.get_text() for label in figure.get_yticklabels()]
assert len(bars) == 2
assert set(labels) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in plotted_data), abs_tol=1e-5)
assert len(figure.get_lines()) == 0
plt.savefig(BytesIO())
# Test with a customized target name.
figure = plot_param_importances(study, target_name="Target Name")
assert len(figure.get_lines()) == 0
assert figure.xaxis.label.get_text() == "Importance for Target Name"
plt.savefig(BytesIO())
# Test with wrong parameters.
with pytest.raises(ValueError):
plot_param_importances(study, params=["optuna"])
# Ignore failed trials.
def fail_objective(_: Trial) -> float:
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError, ))
figure = plot_param_importances(study)
assert len(figure.get_lines()) == 0
plt.savefig(BytesIO())
def test_target_is_none_and_study_is_multi_obj() -> None:
study = create_study(directions=["minimize", "minimize"])
with pytest.raises(ValueError):
plot_param_importances(study)
def _log_plots(run,
study: optuna.Study,
visualization_backend='plotly',
log_plot_contour=True,
log_plot_edf=True,
log_plot_parallel_coordinate=True,
log_plot_param_importances=True,
log_plot_pareto_front=True,
log_plot_slice=True,
log_plot_intermediate_values=True,
log_plot_optimization_history=True,
):
if visualization_backend == 'matplotlib':
import optuna.visualization.matplotlib as vis
elif visualization_backend == 'plotly':
import optuna.visualization as vis
else:
raise NotImplementedError(f'{visualization_backend} visualisation backend is not implemented')
if vis.is_available:
params = list(p_name for t in study.trials for p_name in t.params.keys())
if log_plot_contour and any(params):
run['visualizations/plot_contour'] = neptune.types.File.as_html(vis.plot_contour(study))
if log_plot_edf:
run['visualizations/plot_edf'] = neptune.types.File.as_html(vis.plot_edf(study))
if log_plot_parallel_coordinate:
run['visualizations/plot_parallel_coordinate'] = \
neptune.types.File.as_html(vis.plot_parallel_coordinate(study))
if log_plot_param_importances and len(study.get_trials(states=(optuna.trial.TrialState.COMPLETE, optuna.trial.TrialState.PRUNED,))) > 1:
try:
run['visualizations/plot_param_importances'] = neptune.types.File.as_html(vis.plot_param_importances(study))
except (RuntimeError, ValueError, ZeroDivisionError):
# Unable to compute importances
pass
if log_plot_pareto_front and study._is_multi_objective() and visualization_backend == 'plotly':
run['visualizations/plot_pareto_front'] = neptune.types.File.as_html(vis.plot_pareto_front(study))
if log_plot_slice and any(params):
run['visualizations/plot_slice'] = neptune.types.File.as_html(vis.plot_slice(study))
if log_plot_intermediate_values and any(trial.intermediate_values for trial in study.trials):
# Intermediate values plot if available only if the above condition is met
run['visualizations/plot_intermediate_values'] = \
neptune.types.File.as_html(vis.plot_intermediate_values(study))
if log_plot_optimization_history:
run['visualizations/plot_optimization_history'] = \
neptune.types.File.as_html(vis.plot_optimization_history(study))
