def test_target_is_not_none_and_study_is_multi_obj() -> None: # Multiple sub-figures. study = prepare_study_with_trials(more_than_three=True, n_objectives=2, with_c_d=True) plot_contour(study, target=lambda t: t.values[0]) # Single figure. study = prepare_study_with_trials(more_than_three=True, n_objectives=2, with_c_d=False) plot_contour(study, target=lambda t: t.values[0])
def test_nonfinite_multiobjective(recwarn: WarningsRecorder, objective: int, value: float) -> None: study = prepare_study_with_trials(with_c_d=True, n_objectives=2, value_for_first_trial=value) plot_contour(study, target=lambda t: t.values[objective]) for record in recwarn.list: assert "No contour levels were found within the data range" not in str( record.message) plt.savefig(BytesIO())
def test_nonfinite_removed(recwarn: WarningsRecorder, value: float) -> None: # To check if contour lines have been rendered (meaning +-inf trials were removed), # we should be looking if artists responsible for drawing them are preset in the final plot. # Turns out it's difficult to do reliably (No information which artists are responsible for # drawing contours) so instead we are checking for warning raised by matplotlib # when contour plot fails. TODO(xadrianzetx) Find a better way to test this. study = prepare_study_with_trials(with_c_d=True, value_for_first_trial=value) plot_contour(study) for record in recwarn.list: assert "No contour levels were found within the data range" not in str( record.message) plt.savefig(BytesIO())
def test_plot_contour_mixture_category_types() -> None: study = create_study() study.add_trial( create_trial( value=0.0, params={ "param_a": None, "param_b": 101 }, distributions={ "param_a": CategoricalDistribution([None, "100"]), "param_b": CategoricalDistribution([101, 102.0]), }, )) study.add_trial( create_trial( value=0.5, params={ "param_a": "100", "param_b": 102.0 }, distributions={ "param_a": CategoricalDistribution([None, "100"]), "param_b": CategoricalDistribution([101, 102.0]), }, )) figure = plot_contour(study) assert figure.get_xlim() == (-0.05, 1.05) assert figure.get_ylim() == (100.95, 102.05)
def test_plot_contour_customized_target_name(params: List[str]) -> None: study = prepare_study_with_trials(more_than_three=True) figure = plot_contour(study, params=params, target_name="Target Name") if len(params) == 2: assert figure.has_data() else: assert figure.shape == (len(params), len(params))
def test_plot_contour_customized_target(params: List[str]) -> None: study = prepare_study_with_trials(more_than_three=True) with pytest.warns(UserWarning): figure = plot_contour(study, params=params, target=lambda t: t.params["param_d"]) if len(params) == 2: assert figure.has_data() else: assert figure.shape == (len(params), len(params))
def test_plot_contour(params: Optional[List[str]]) -> None: # Test with no trial. study_without_trials = prepare_study_with_trials(no_trials=True) figure = plot_contour(study_without_trials, params=params) assert len(figure.get_lines()) == 0 plt.savefig(BytesIO()) # Test whether trials with `ValueError`s are ignored. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError, )) figure = plot_contour(study, params=params) assert len(figure.get_lines()) == 0 plt.savefig(BytesIO()) # Test with some trials. study = prepare_study_with_trials() # Test ValueError due to wrong params. with pytest.raises(ValueError): plot_contour(study, ["optuna", "Optuna"]) figure = plot_contour(study, params=params) if params is not None and len(params) < 3: if len(params) <= 1: assert len(figure.get_lines()) == 0 elif len(params) == 2: assert len(figure.get_lines()) == 0 elif params is None: assert figure.shape == (len(study.best_params), len(study.best_params)) for i in range(len(study.best_params)): assert figure[i][0].yaxis.label.get_text() == list( study.best_params)[i] else: assert figure.shape == (len(params), len(params)) for i in range(len(params)): assert figure[i][0].yaxis.label.get_text() == list(params)[i] plt.savefig(BytesIO())
def test_plot_contour_customized_target_name(params: List[str]) -> None: study = prepare_study_with_trials() figure = plot_contour(study, params=params, target_name="Target Name") if len(params) == 2: assert len(figure.get_lines()) == 0 else: assert figure.shape == (len(params), len(params)) for i in range(len(params)): assert figure[i][0].yaxis.label.get_text() == list(params)[i]
def _log_plots(run, study: optuna.Study, visualization_backend='plotly', log_plot_contour=True, log_plot_edf=True, log_plot_parallel_coordinate=True, log_plot_param_importances=True, log_plot_pareto_front=True, log_plot_slice=True, log_plot_intermediate_values=True, log_plot_optimization_history=True, ): if visualization_backend == 'matplotlib': import optuna.visualization.matplotlib as vis elif visualization_backend == 'plotly': import optuna.visualization as vis else: raise NotImplementedError(f'{visualization_backend} visualisation backend is not implemented') if vis.is_available: params = list(p_name for t in study.trials for p_name in t.params.keys()) if log_plot_contour and any(params): run['visualizations/plot_contour'] = neptune.types.File.as_html(vis.plot_contour(study)) if log_plot_edf: run['visualizations/plot_edf'] = neptune.types.File.as_html(vis.plot_edf(study)) if log_plot_parallel_coordinate: run['visualizations/plot_parallel_coordinate'] = \ neptune.types.File.as_html(vis.plot_parallel_coordinate(study)) if log_plot_param_importances and len(study.get_trials(states=(optuna.trial.TrialState.COMPLETE, optuna.trial.TrialState.PRUNED,))) > 1: try: run['visualizations/plot_param_importances'] = neptune.types.File.as_html(vis.plot_param_importances(study)) except (RuntimeError, ValueError, ZeroDivisionError): # Unable to compute importances pass if log_plot_pareto_front and study._is_multi_objective() and visualization_backend == 'plotly': run['visualizations/plot_pareto_front'] = neptune.types.File.as_html(vis.plot_pareto_front(study)) if log_plot_slice and any(params): run['visualizations/plot_slice'] = neptune.types.File.as_html(vis.plot_slice(study)) if log_plot_intermediate_values and any(trial.intermediate_values for trial in study.trials): # Intermediate values plot if available only if the above condition is met run['visualizations/plot_intermediate_values'] = \ neptune.types.File.as_html(vis.plot_intermediate_values(study)) if log_plot_optimization_history: run['visualizations/plot_optimization_history'] = \ neptune.types.File.as_html(vis.plot_optimization_history(study))
def test_plot_contour_customized_target(params: List[str]) -> None: study = prepare_study_with_trials() with pytest.warns(UserWarning): figure = plot_contour(study, params=params, target=lambda t: t.params["param_d"]) if len(params) == 2: assert len(figure.get_lines()) == 0 else: assert figure.shape == (len(params), len(params)) for i in range(len(params)): assert figure[i][0].yaxis.label.get_text() == list(params)[i]
def test_plot_contour_log_scale_and_str_category() -> None: # If the search space has three parameters, plot_contour generates nine plots. study = create_study() study.add_trial( create_trial( value=0.0, params={ "param_a": 1e-6, "param_b": "100", "param_c": "one" }, distributions={ "param_a": FloatDistribution(1e-7, 1e-2, log=True), "param_b": CategoricalDistribution(["100", "101"]), "param_c": CategoricalDistribution(["one", "two"]), }, )) study.add_trial( create_trial( value=1.0, params={ "param_a": 1e-5, "param_b": "101", "param_c": "two" }, distributions={ "param_a": FloatDistribution(1e-7, 1e-2, log=True), "param_b": CategoricalDistribution(["100", "101"]), "param_c": CategoricalDistribution(["one", "two"]), }, )) figure = plot_contour(study) subplots = [plot for plot in figure.flatten() if plot.has_data()] expected = { "param_a": [1e-6, 1e-5], "param_b": [0.0, 1.0], "param_c": [0.0, 1.0] } ranges = itertools.permutations(expected.keys(), 2) for plot, (yrange, xrange) in zip(subplots, ranges): # Take 5% axis padding into account. np.testing.assert_allclose(plot.get_xlim(), expected[xrange], atol=5e-2) np.testing.assert_allclose(plot.get_ylim(), expected[yrange], atol=5e-2) plt.savefig(BytesIO())
def test_plot_contour(params: Optional[List[str]]) -> None: # Test with no trial. study_without_trials = prepare_study_with_trials(no_trials=True) figure = plot_contour(study_without_trials, params=params) assert not figure.has_data() # Test whether trials with `ValueError`s are ignored. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError, )) figure = plot_contour(study, params=params) assert not figure.has_data() # Test with some trials. study = prepare_study_with_trials(more_than_three=True) # Test ValueError due to wrong params. with pytest.raises(ValueError): plot_contour(study, ["optuna", "Optuna"]) figure = plot_contour(study, params=params) if params is not None and len(params) < 3: if len(params) <= 1: assert not figure.has_data() elif len(params) == 2: # TODO(ytknzw): Add more specific assertion with the test case. assert figure.has_data() elif params is None: # TODO(ytknzw): Add more specific assertion with the test case. assert figure.shape == (len(study.best_params), len(study.best_params)) else: # TODO(ytknzw): Add more specific assertion with the test case. assert figure.shape == (len(params), len(params))
def test_contour_subplots_have_correct_axis_labels_and_ranges() -> None: study = prepare_study_with_trials() params = ["param_a", "param_b", "param_c"] subplots = plot_contour(study, params=params) # `subplots` should look like this: # param_a [[subplot 1, subplot 2, subplot 3], # param_b [subplot 4, subplot 4, subplot 6], # param_c [subplot 7, subplot 8, subplot 9]] # param_a param_b param_c # # The folowing block ensures: # - The y-axis label of subplot 1 is "param_a" # - The x-axis label of subplot 7 is "param_a" # - Subplot 1, 2, and 3 have the same y-axis range that covers the search space for `param_a` # - Subplot 1, 4, and 7 have the same x-axis range that covers the search space for `param_a` # - The y-axis label of subplot 4 is "param_b" # - ... # - The y-axis label of subplot 7 is "param_c" # - ... param_ranges = { "param_a": (0.0, 3.0), "param_b": (0.0, 3.0), "param_c": (2.0, 5.0), } for index, (param_name, param_range) in enumerate(param_ranges.items()): minimum, maximum = param_range padding = (maximum - minimum) * AXES_PADDING_RATIO param_range_with_padding = (minimum - padding, maximum + padding) assert subplots[index, 0].get_ylabel() == param_name assert subplots[-1, index].get_xlabel() == param_name ylims = [ax.get_ylim() for ax in subplots[index, :]] assert all_equal(ylims) assert all( range_covers(param_range_with_padding, ylim) for ylim in ylims) xlims = [ax.get_xlim() for ax in subplots[:, index]] assert all_equal(xlims) assert all( range_covers(param_range_with_padding, xlim) for xlim in xlims) plt.savefig(BytesIO())
def test_target_is_none_and_study_is_multi_obj() -> None: study = create_study(directions=["minimize", "minimize"]) with pytest.raises(ValueError): plot_contour(study)
def test_generate_contour_plot_for_few_observations(params: List[str]) -> None: study = prepare_study_with_trials(less_than_two=True) figure = plot_contour(study, params) assert not figure.has_data()