def test_datetime_start(storage_init_func): # type: (Callable[[], storages.BaseStorage]) -> None trial_datetime_start = [None] # type: List[Optional[datetime.datetime]] def objective(trial): # type: (Trial) -> float trial_datetime_start[0] = trial.datetime_start return 1.0 study = create_study(storage_init_func()) study.optimize(objective, n_trials=1) assert study.trials[0].datetime_start == trial_datetime_start[0]
def test_get_param_importances_invalid_empty_study( evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None: study = create_study() with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func()) def objective(trial: Trial) -> float: raise optuna.TrialPruned study.optimize(objective, n_trials=3) with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func())
def test_check_distribution_suggest_int( storage_init_func: Callable[[], storages.BaseStorage], enable_log: bool ) -> None: sampler = samplers.RandomSampler() study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(None) as record: trial.suggest_int("x", 10, 20, log=enable_log) trial.suggest_int("x", 10, 20, log=enable_log) trial.suggest_int("x", 10, 22, log=enable_log) # We expect exactly one warning. assert len(record) == 1
def test_optuna_search_study_with_minimize() -> None: X, y = make_blobs(n_samples=10) est = KernelDensity() study = create_study(direction="minimize") optuna_search = integration.OptunaSearchCV(est, {}, cv=3, error_score="raise", random_state=0, return_train_score=True, study=study) with pytest.raises(ValueError, match="direction of study must be 'maximize'."): optuna_search.fit(X)
def test_get_info_importances_nonfinite_removed( inf_value: float, evaluator: BaseImportanceEvaluator, n_trials: int) -> None: def _objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 2, 4, log=True) return x1 + x2 * x3 seed = 13 target_name = "Objective Value" study = create_study(sampler=RandomSampler(seed=seed)) study.optimize(_objective, n_trials=n_trials) # Create param importances info without inf value. info_without_inf = _get_importances_info(study, evaluator=evaluator, params=None, target=None, target_name=target_name) # A trial with an inf value is added into the study manually. study.add_trial( create_trial( value=inf_value, params={ "x1": 1.0, "x2": 1.0, "x3": 3.0 }, distributions={ "x1": FloatDistribution(low=0.1, high=3), "x2": FloatDistribution(low=0.1, high=3, log=True), "x3": FloatDistribution(low=2, high=4, log=True), }, )) # Create param importances info with inf value. info_with_inf = _get_importances_info(study, evaluator=evaluator, params=None, target=None, target_name=target_name) # Obtained info instances should be the same between with inf and without inf, # because the last trial whose objective value is an inf is ignored. assert info_with_inf == info_without_inf
def test_plot_slice() -> None: # Test with no trial. study = prepare_study_with_trials(no_trials=True) figure = plot_slice(study) assert len(figure.data) == 0 study = prepare_study_with_trials(with_c_d=False) # Test with a trial. figure = plot_slice(study) assert len(figure.data) == 2 assert figure.data[0]["x"] == (1.0, 2.5) assert figure.data[0]["y"] == (0.0, 1.0) assert figure.data[1]["x"] == (2.0, 0.0, 1.0) assert figure.data[1]["y"] == (0.0, 2.0, 1.0) assert figure.layout.yaxis.title.text == "Objective Value" # Test with a trial to select parameter. figure = plot_slice(study, params=["param_a"]) assert len(figure.data) == 1 assert figure.data[0]["x"] == (1.0, 2.5) assert figure.data[0]["y"] == (0.0, 1.0) # Test with a customized target value. figure = plot_slice(study, params=["param_a"], target=lambda t: t.params["param_b"]) assert len(figure.data) == 1 assert figure.data[0]["x"] == (1.0, 2.5) assert figure.data[0]["y"] == (2.0, 1.0) assert figure.layout.yaxis.title.text == "Objective Value" # Test with a customized target name. figure = plot_slice(study, target_name="Target Name") assert figure.layout.yaxis.title.text == "Target Name" # Test with wrong parameters. with pytest.raises(ValueError): plot_slice(study, params=["optuna"]) # Ignore failed trials. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError,)) figure = plot_slice(study) assert len(figure.data) == 0
def test_generate_contour_plot_for_few_observations(params: List[str]) -> None: study = create_study(direction="minimize") study.add_trial( create_trial( values=[0.0], params={ "param_a": 1.0, "param_b": 2.0 }, distributions={ "param_a": FloatDistribution(0.0, 3.0), "param_b": FloatDistribution(0.0, 3.0), }, )) study.add_trial( create_trial( values=[2.0], params={"param_b": 0.0}, distributions={"param_b": FloatDistribution(0.0, 3.0)}, )) info = _get_contour_info(study, params=params) assert info == _ContourInfo( sorted_params=sorted(params), sub_plot_infos=[[ _SubContourInfo( xaxis=_AxisInfo( name=sorted(params)[0], range=(1.0, 1.0), is_log=False, is_cat=False, indices=[1.0], values=[1.0, None], ), yaxis=_AxisInfo( name=sorted(params)[1], range=(-0.1, 2.1), is_log=False, is_cat=False, indices=[-0.1, 0.0, 2.0, 2.1], values=[2.0, 0.0], ), z_values={}, ) ]], reverse_scale=True, target_name="Objective Value", )
def test_plot_parallel_coordinate_log_params() -> None: # Test with log params. study_log_params = create_study() study_log_params.add_trial( create_trial( value=0.0, params={"param_a": 1e-6, "param_b": 10}, distributions={ "param_a": LogUniformDistribution(1e-7, 1e-2), "param_b": LogUniformDistribution(1, 1000), }, ) ) study_log_params.add_trial( create_trial( value=1.0, params={"param_a": 2e-5, "param_b": 200}, distributions={ "param_a": LogUniformDistribution(1e-7, 1e-2), "param_b": LogUniformDistribution(1, 1000), }, ) ) study_log_params.add_trial( create_trial( value=0.1, params={"param_a": 1e-4, "param_b": 30}, distributions={ "param_a": LogUniformDistribution(1e-7, 1e-2), "param_b": LogUniformDistribution(1, 1000), }, ) ) figure = plot_parallel_coordinate(study_log_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0, 0.1) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (-6.0, -4.0) assert figure.data[0]["dimensions"][1]["values"] == (-6, math.log10(2e-5), -4) assert figure.data[0]["dimensions"][1]["ticktext"] == ("1e-06", "1e-05", "0.0001") assert figure.data[0]["dimensions"][1]["tickvals"] == (-6, -5, -4.0) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (1.0, math.log10(200)) assert figure.data[0]["dimensions"][2]["values"] == (1.0, math.log10(200), math.log10(30)) assert figure.data[0]["dimensions"][2]["ticktext"] == ("10", "100", "200") assert figure.data[0]["dimensions"][2]["tickvals"] == (1.0, 2.0, math.log10(200))
def test_plot_edf_with_target_name(plot_edf: Callable[..., Any], target_name: Optional[str]) -> None: study = create_study() study.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10) if target_name is None: figure = plot_edf(study) else: figure = plot_edf(study, target_name=target_name) expected = target_name if target_name is not None else "Objective Value" if isinstance(figure, go.Figure): assert figure.layout.xaxis.title.text == expected elif isinstance(figure, Axes): assert figure.xaxis.label.get_text() == expected save_static_image(figure)
def test_switch_label_when_param_insignificant() -> None: def _objective(trial: Trial) -> int: x = trial.suggest_int("x", 0, 2) _ = trial.suggest_int("y", -1, 1) return x**2 study = create_study() for x in range(1, 3): study.enqueue_trial({"x": x, "y": 0}) study.optimize(_objective, n_trials=2) ax = plot_param_importances(study) # Test if label for `y` param has been switched to `<0.01`. labels = ax.figure.findobj(lambda obj: "<0.01" in str(obj)) assert len(labels) == 1
def test_switch_label_when_param_insignificant() -> None: def _objective(trial: Trial) -> int: x = trial.suggest_int("x", 0, 2) _ = trial.suggest_int("y", -1, 1) return x**2 study = create_study() for x in range(1, 3): study.enqueue_trial({"x": x, "y": 0}) study.optimize(_objective, n_trials=2) figure = plot_param_importances(study) # Test if label for `y` param has been switched to `<0.01`. labels = figure.data[0].text assert labels == ("<0.01", "1.00")
def test_suggest_int(storage_init_func: Callable[[], storages.BaseStorage]) -> None: mock = Mock() mock.side_effect = [1, 2] sampler = samplers.RandomSampler() with patch.object(sampler, "sample_independent", mock) as mock_object: study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) distribution = IntUniformDistribution(low=0, high=3) assert trial._suggest("x", distribution) == 1 # Test suggesting a param. assert trial._suggest("x", distribution) == 1 # Test suggesting the same param. assert trial._suggest("y", distribution) == 2 # Test suggesting a different param. assert trial.params == {"x": 1, "y": 2} assert mock_object.call_count == 2
def test_empty_edf_info() -> None: def _assert_empty(info: _EDFInfo) -> None: assert info.lines == [] np.testing.assert_array_equal(info.x_values, np.array([])) edf_info = _get_edf_info([]) _assert_empty(edf_info) study = create_study() edf_info = _get_edf_info(study) _assert_empty(edf_info) trial = study.ask() study.tell(trial, state=optuna.trial.TrialState.PRUNED) edf_info = _get_edf_info(study) _assert_empty(edf_info)
def test_check_distribution_suggest_float(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None sampler = samplers.RandomSampler() study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) x1 = trial.suggest_float("x1", 10, 20) x2 = trial.suggest_uniform("x1", 10, 20) assert x1 == x2 x3 = trial.suggest_float("x2", 1e-5, 1e-3, log=True) x4 = trial.suggest_loguniform("x2", 1e-5, 1e-3) assert x3 == x4
def test_switch_label_when_param_insignificant() -> None: def _objective(trial: Trial) -> int: x = trial.suggest_int("x", 0, 2) _ = trial.suggest_int("y", -1, 1) return x**2 study = create_study() for x in range(1, 3): study.enqueue_trial({"x": x, "y": 0}) study.optimize(_objective, n_trials=2) info = _get_importances_info(study, None, None, None, "Objective Value") # Test if label for `y` param has been switched to `<0.01`. assert info.importance_labels == ["<0.01", "1.00"]
def test_relative_parameters(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None relative_search_space = { 'x': distributions.UniformDistribution(low=5, high=6), 'y': distributions.UniformDistribution(low=5, high=6) } relative_params = { 'x': 5.5, 'y': 5.5, 'z': 5.5 } sampler = DeterministicRelativeSampler(relative_search_space, relative_params) # type: ignore study = create_study(storage=storage_init_func(), sampler=sampler) def create_trial(): # type: () -> Trial return Trial(study, study.storage.create_new_trial_id(study.study_id)) # Suggested from `relative_params`. trial0 = create_trial() distribution0 = distributions.UniformDistribution(low=0, high=100) assert trial0._suggest('x', distribution0) == 5.5 # Not suggested from `relative_params` (due to unknown parameter name). trial1 = create_trial() distribution1 = distribution0 assert trial1._suggest('w', distribution1) != 5.5 # Not suggested from `relative_params` (due to incompatible value range). trial2 = create_trial() distribution2 = distributions.UniformDistribution(low=0, high=5) assert trial2._suggest('x', distribution2) != 5.5 # Error (due to incompatible distribution class). trial3 = create_trial() distribution3 = distributions.IntUniformDistribution(low=1, high=100) with pytest.raises(ValueError): trial3._suggest('y', distribution3) # Error ('z' is included in `relative_params` but not in `relative_search_space`). trial4 = create_trial() distribution4 = distributions.UniformDistribution(low=0, high=10) with pytest.raises(ValueError): trial4._suggest('z', distribution4)
def test_relative_parameters(storage_init_func: Callable[[], storages.BaseStorage]) -> None: relative_search_space = { "x": UniformDistribution(low=5, high=6), "y": UniformDistribution(low=5, high=6), } relative_params = {"x": 5.5, "y": 5.5, "z": 5.5} sampler = DeterministicRelativeSampler(relative_search_space, relative_params) # type: ignore study = create_study(storage=storage_init_func(), sampler=sampler) def create_trial() -> Trial: return Trial(study, study._storage.create_new_trial(study._study_id)) # Suggested from `relative_params`. trial0 = create_trial() distribution0 = UniformDistribution(low=0, high=100) assert trial0._suggest("x", distribution0) == 5.5 # Not suggested from `relative_params` (due to unknown parameter name). trial1 = create_trial() distribution1 = distribution0 assert trial1._suggest("w", distribution1) != 5.5 # Not suggested from `relative_params` (due to incompatible value range). trial2 = create_trial() distribution2 = UniformDistribution(low=0, high=5) assert trial2._suggest("x", distribution2) != 5.5 # Error (due to incompatible distribution class). trial3 = create_trial() distribution3 = IntUniformDistribution(low=1, high=100) with pytest.raises(ValueError): trial3._suggest("y", distribution3) # Error ('z' is included in `relative_params` but not in `relative_search_space`). trial4 = create_trial() distribution4 = UniformDistribution(low=0, high=10) with pytest.raises(ValueError): trial4._suggest("z", distribution4) # Error (due to incompatible distribution class). trial5 = create_trial() distribution5 = IntLogUniformDistribution(low=1, high=100) with pytest.raises(ValueError): trial5._suggest("y", distribution5)
def test_plot_parallel_coordinate_log_params() -> None: # Test with log params. study_log_params = create_study() distributions: Dict[str, BaseDistribution] = { "param_a": FloatDistribution(1e-7, 1e-2, log=True), "param_b": FloatDistribution(1, 1000, log=True), } study_log_params.add_trial( create_trial( value=0.0, params={ "param_a": 1e-6, "param_b": 10 }, distributions=distributions, )) study_log_params.add_trial( create_trial( value=1.0, params={ "param_a": 2e-5, "param_b": 200 }, distributions=distributions, )) study_log_params.add_trial( create_trial( value=0.1, params={ "param_a": 1e-4, "param_b": 30 }, distributions=distributions, )) figure = plot_parallel_coordinate(study_log_params) axes = figure.get_figure().axes assert len(axes) == 3 + 1 assert axes[0].get_ylim() == (0.0, 1.0) assert axes[1].get_ylabel() == "Objective Value" assert axes[1].get_ylim() == (0.0, 1.0) objectives = _fetch_objectives_from_figure(figure) assert objectives == [0.0, 1.0, 0.1] assert axes[2].get_ylim() == (1e-6, 1e-4) np.testing.assert_almost_equal(axes[3].get_ylim(), (10.0, 200)) expected_labels = ["Objective Value", "param_a", "param_b"] _test_xtick_labels(axes, expected_labels) plt.savefig(BytesIO())
def test_plot_slice() -> None: # Test with no trial. study = prepare_study_with_trials(no_trials=True) figure = plot_slice(study) assert not figure.has_data() study = prepare_study_with_trials(with_c_d=False) # Test with a trial. # TODO(ytknzw): Add more specific assertion with the test case. figure = plot_slice(study) assert len(figure) == 2 assert figure[0].has_data() assert figure[1].has_data() # Test with a trial to select parameter. # TODO(ytknzw): Add more specific assertion with the test case. figure = plot_slice(study, params=["param_a"]) assert figure.has_data() # Test with a customized target value. with pytest.warns(UserWarning): figure = plot_slice(study, params=["param_a"], target=lambda t: t.params["param_b"]) assert figure.has_data() # Test with a customized target name. figure = plot_slice(study, target_name="Target Name") assert len(figure) == 2 assert figure[0].has_data() assert figure[1].has_data() # Test with wrong parameters. with pytest.raises(ValueError): plot_slice(study, params=["optuna"]) # Ignore failed trials. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError, )) figure = plot_slice(study) assert not figure.has_data()
def test_plot_slice() -> None: # Test with no trial. study = prepare_study_with_trials(no_trials=True) figure = plot_slice(study) assert len(figure.get_lines()) == 0 study = prepare_study_with_trials(with_c_d=False) # Test with a trial. figure = plot_slice(study) assert len(figure) == 2 assert len(figure[0].get_lines()) == 0 assert len(figure[1].get_lines()) == 0 assert figure[0].yaxis.label.get_text() == "Objective Value" # Test with a trial to select parameter. figure = plot_slice(study, params=["param_a"]) assert len(figure.get_lines()) == 0 assert figure.yaxis.label.get_text() == "Objective Value" # Test with a customized target value. with pytest.warns(UserWarning): figure = plot_slice(study, params=["param_a"], target=lambda t: t.params["param_b"]) assert len(figure.get_lines()) == 0 assert figure.yaxis.label.get_text() == "Objective Value" # Test with a customized target name. figure = plot_slice(study, target_name="Target Name") assert len(figure) == 2 assert len(figure[0].get_lines()) == 0 assert len(figure[1].get_lines()) == 0 assert figure[0].yaxis.label.get_text() == "Target Name" # Test with wrong parameters. with pytest.raises(ValueError): plot_slice(study, params=["optuna"]) # Ignore failed trials. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError,)) figure = plot_slice(study) assert len(figure.get_lines()) == 0
def test_plot_optimization_history(direction: str) -> None: # Test with no studies. figure = plot_edf([]) assert len(figure.data) == 0 # Test with no trials. figure = plot_edf(create_study(direction=direction)) assert len(figure.data) == 0 figure = plot_edf( [create_study(direction=direction), create_study(direction=direction)]) assert len(figure.data) == 0 # Test with a study. study0 = create_study(direction=direction) study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10) figure = plot_edf(study0) _validate_edf_values(figure.data[0]["y"]) assert len(figure.data) == 1 assert figure.layout.xaxis.title.text == "Objective Value" # Test with two studies. study1 = create_study(direction=direction) study1.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10) figure = plot_edf([study0, study1]) for points in figure.data: _validate_edf_values(points["y"]) assert len(figure.data) == 2 figure = plot_edf((study0, study1)) for points in figure.data: _validate_edf_values(points["y"]) assert len(figure.data) == 2 # Test with a customized target value. study0 = create_study(direction=direction) study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10) with pytest.warns(UserWarning): figure = plot_edf(study0, target=lambda t: t.params["x"]) _validate_edf_values(figure.data[0]["y"]) assert len(figure.data) == 1 # Test with a customized target name. study0 = create_study(direction=direction) study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10) figure = plot_edf(study0, target_name="Target Name") _validate_edf_values(figure.data[0]["y"]) assert len(figure.data) == 1 assert figure.layout.xaxis.title.text == "Target Name"
def test_get_param_importances_empty_search_space( evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None: def objective(trial: Trial) -> float: x = trial.suggest_float("x", 0, 5) y = trial.suggest_float("y", 1, 1) return 4 * x**2 + 4 * y**2 study = create_study() study.optimize(objective, n_trials=3) param_importance = get_param_importances(study, evaluator=evaluator_init_func()) assert len(param_importance) == 2 assert all([param in param_importance for param in ["x", "y"]]) assert param_importance["x"] > 0.0 assert param_importance["y"] == 0.0
def test_suggest_low_equals_high(storage_init_func): # type: (Callable[[], storages.BaseStorage]) -> None study = create_study(storage_init_func(), sampler=samplers.TPESampler(n_startup_trials=0)) trial = Trial(study, study._storage.create_new_trial(study._study_id)) # Parameter values are determined without suggestion when low == high. with patch.object(trial, "_suggest", wraps=trial._suggest) as mock_object: assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting a param. assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting a param. assert trial.suggest_loguniform( "b", 1.0, 1.0) == 1.0 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_discrete_uniform( "c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param. assert (trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0 ) # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param. assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting a param. assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("f", 0.5, 0.5, log=True) == 0.5 # Suggesting a param. assert trial.suggest_float( "f", 0.5, 0.5, log=True) == 0.5 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("g", 0.5, 0.5, log=False) == 0.5 # Suggesting a param. assert trial.suggest_float( "g", 0.5, 0.5, log=False) == 0.5 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting a param. assert trial.suggest_float( "h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting the same param. assert mock_object.call_count == 0
def _create_study( mo_study: "multi_objective.study.MultiObjectiveStudy" ) -> "optuna.Study": study = create_study( storage=mo_study._storage, sampler=_MultiObjectiveSamplerAdapter(mo_study.sampler), pruner=NopPruner(), study_name="_motpe-" + mo_study._storage.get_study_name_from_id(mo_study._study_id), directions=mo_study.directions, load_if_exists=True, ) for mo_trial in mo_study.trials: with warnings.catch_warnings(): warnings.simplefilter("ignore", ExperimentalWarning) study.add_trial(_create_trial(mo_trial)) return study
def test_suggest_discrete_uniform(storage_init_func): # type: (Callable[[], storages.BaseStorage]) -> None mock = Mock() mock.side_effect = [1.0, 2.0, 3.0] sampler = samplers.RandomSampler() with patch.object(sampler, "sample_independent", mock) as mock_object: study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) distribution = DiscreteUniformDistribution(low=0.0, high=3.0, q=1.0) assert trial._suggest("x", distribution) == 1.0 # Test suggesting a param. assert trial._suggest("x", distribution) == 1.0 # Test suggesting the same param. assert trial._suggest("y", distribution) == 3.0 # Test suggesting a different param. assert trial.params == {"x": 1.0, "y": 3.0} assert mock_object.call_count == 3
def test_trial_with_infinite_value_ignored(inf_value: float, evaluator: BaseImportanceEvaluator, n_trial: int) -> None: def _objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 2, 4, log=True) return x1 + x2 * x3 seed = 13 study = create_study(sampler=RandomSampler(seed=seed)) study.optimize(_objective, n_trials=n_trial) # A figure is created without a trial with an inf value. plot_param_importances(study, evaluator=evaluator) with BytesIO() as byte_io: plt.savefig(byte_io) figure_with_inf = byte_io.getvalue() # A trial with an inf value is added into the study manually. study.add_trial( create_trial( value=inf_value, params={ "x1": 1.0, "x2": 1.0, "x3": 3.0 }, distributions={ "x1": FloatDistribution(low=0.1, high=3), "x2": FloatDistribution(low=0.1, high=3, log=True), "x3": FloatDistribution(low=2, high=4, log=True), }, )) # A figure is created with a trial with an inf value. plot_param_importances(study, evaluator=evaluator) with BytesIO() as byte_io: plt.savefig(byte_io) figure_without_inf = byte_io.getvalue() # Obtained figures should be the same between with inf and without inf, # because the last trial whose objective value is an inf is ignored. assert len(figure_without_inf) > 0 assert figure_with_inf == figure_without_inf
def test_error_bar_in_optimization_history(direction: str) -> None: def objective(trial: Trial) -> float: return trial.suggest_float("x", 0, 1) studies = [create_study(direction=direction) for _ in range(3)] suggested_params = [0.1, 0.3, 0.2] for x, study in zip(suggested_params, studies): study.enqueue_trial({"x": x}) study.optimize(objective, n_trials=1) figure = plot_optimization_history(studies, error_bar=True) mean = np.mean(suggested_params) std = np.std(suggested_params) np.testing.assert_almost_equal(figure.data[0].y, mean) np.testing.assert_almost_equal(figure.data[2].y, mean + std) np.testing.assert_almost_equal(figure.data[3].y, mean - std)
def test_suggest_discrete_uniform(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None mock = Mock() mock.side_effect = [1., 2., 3.] sampler = samplers.RandomSampler() with patch.object(sampler, 'sample_independent', mock) as mock_object: study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study.storage.create_new_trial_id(study.study_id)) distribution = distributions.DiscreteUniformDistribution(low=0., high=3., q=1.) assert trial._suggest('x', distribution) == 1. # Test suggesting a param. assert trial._suggest('x', distribution) == 1. # Test suggesting the same param. assert trial._suggest('y', distribution) == 3. # Test suggesting a different param. assert trial.params == {'x': 1., 'y': 3.} assert mock_object.call_count == 3
def test_get_optimization_history_info_list_with_error_bar( direction: str, target_name: str) -> None: n_studies = 10 def objective(trial: Trial) -> float: if trial.number == 0: return 1.0 elif trial.number == 1: return 2.0 elif trial.number == 2: return 0.0 return 0.0 # Test with trials. studies = [create_study(direction=direction) for _ in range(n_studies)] for study in studies: study.optimize(objective, n_trials=3) info_list = _get_optimization_history_info_list(study, target=None, target_name=target_name, error_bar=True) best_values = [1.0, 1.0, 0.0 ] if direction == "minimize" else [1.0, 2.0, 2.0] assert info_list == [ _OptimizationHistoryInfo( [0, 1, 2], _ValuesInfo([1.0, 2.0, 0.0], [0.0, 0.0, 0.0], target_name), _ValuesInfo(best_values, [0.0, 0.0, 0.0], "Best Value"), ) ] # Test customized target. info_list = _get_optimization_history_info_list(study, target=lambda t: t.number, target_name=target_name, error_bar=True) assert info_list == [ _OptimizationHistoryInfo( [0, 1, 2], _ValuesInfo([0.0, 1.0, 2.0], [0.0, 0.0, 0.0], target_name), None, ) ]
def test_plot_parallel_coordinate_categorical_numeric_params() -> None: # Test with categorical params that can be interpreted as numeric params. study_categorical_params = create_study() distributions: Dict[str, BaseDistribution] = { "category_a": CategoricalDistribution((1, 2)), "category_b": CategoricalDistribution((10, 20, 30)), } study_categorical_params.add_trial( create_trial( value=0.0, params={"category_a": 2, "category_b": 20}, distributions=distributions, ) ) study_categorical_params.add_trial( create_trial( value=1.0, params={"category_a": 1, "category_b": 30}, distributions=distributions, ) ) study_categorical_params.add_trial( create_trial( value=2.0, params={"category_a": 2, "category_b": 10}, distributions=distributions, ) ) # Trials are sorted by using param_a and param_b, i.e., trial#1, trial#2, and trial#0. figure = plot_parallel_coordinate(study_categorical_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (1.0, 2.0, 0.0) assert figure.data[0]["dimensions"][1]["label"] == "category_a" assert figure.data[0]["dimensions"][1]["range"] == (0, 1) assert figure.data[0]["dimensions"][1]["values"] == (0, 1, 1) assert figure.data[0]["dimensions"][1]["ticktext"] == (1, 2) assert figure.data[0]["dimensions"][1]["tickvals"] == (0, 1) assert figure.data[0]["dimensions"][2]["label"] == "category_b" assert figure.data[0]["dimensions"][2]["range"] == (0, 2) assert figure.data[0]["dimensions"][2]["values"] == (2, 0, 1) assert figure.data[0]["dimensions"][2]["ticktext"] == (10, 20, 30) assert figure.data[0]["dimensions"][2]["tickvals"] == (0, 1, 2)