Ejemplo n.º 1
0
def test_suggest_categorical(trial_type: type) -> None:

    # Integer categories.
    trial = _create_trial(
        trial_type=trial_type,
        params={"x": 1},
        distributions={"x": CategoricalDistribution((0, 1, 2, 3))},
    )
    assert trial.suggest_categorical("x", (0, 1, 2, 3)) == 1

    with pytest.raises(ValueError):
        trial.suggest_categorical("y", [0, 1, 2, 3])

    # String categories.
    trial = _create_trial(
        trial_type=trial_type,
        params={"x": "baz"},
        distributions={"x": CategoricalDistribution(("foo", "bar", "baz"))},
    )
    assert trial.suggest_categorical("x", ("foo", "bar", "baz")) == "baz"

    # Unknown parameter.
    with pytest.raises(ValueError):
        trial.suggest_categorical("y", ["foo", "bar", "baz"])

    # Not in choices.
    with pytest.raises(ValueError):
        trial.suggest_categorical("x", ["foo", "bar"])

    # Unknown parameter and bad category type.
    with pytest.warns(UserWarning):
        with pytest.raises(
                ValueError
        ):  # Must come after `pytest.warns` to catch failures.
            trial.suggest_categorical("x", [{"foo": "bar"}])  # type: ignore
Ejemplo n.º 2
0
def test_search_space_transform_encoding() -> None:
    trans = _SearchSpaceTransform({"x0": IntUniformDistribution(0, 3)})

    assert len(trans.column_to_encoded_columns) == 1
    numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0]))
    numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0]))

    trans = _SearchSpaceTransform({"x0": CategoricalDistribution(["foo", "bar", "baz"])})

    assert len(trans.column_to_encoded_columns) == 1
    numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0, 1, 2]))
    numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0, 0, 0]))

    trans = _SearchSpaceTransform(
        {
            "x0": UniformDistribution(0, 3),
            "x1": CategoricalDistribution(["foo", "bar", "baz"]),
            "x3": DiscreteUniformDistribution(0, 1, q=0.2),
        }
    )

    assert len(trans.column_to_encoded_columns) == 3
    numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0]))
    numpy.testing.assert_equal(trans.column_to_encoded_columns[1], numpy.array([1, 2, 3]))
    numpy.testing.assert_equal(trans.column_to_encoded_columns[2], numpy.array([4]))
    numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0, 1, 1, 1, 2]))
def create_optuna_distribution_from_override(override: Override) -> Any:
    value = override.value()
    if not override.is_sweep_override():
        return value

    if override.is_choice_sweep():
        assert isinstance(value, ChoiceSweep)
        choices = [
            x for x in override.sweep_iterator(transformer=Transformer.encode)
        ]
        return CategoricalDistribution(choices)

    if override.is_range_sweep():
        choices = [
            x for x in override.sweep_iterator(transformer=Transformer.encode)
        ]
        return CategoricalDistribution(choices)

    if override.is_interval_sweep():
        assert isinstance(value, IntervalSweep)
        if "log" in value.tags:
            if "int" in value.tags:
                return IntLogUniformDistribution(value.start, value.end)
            return LogUniformDistribution(value.start, value.end)
        else:
            if "int" in value.tags:
                return IntUniformDistribution(value.start, value.end)
            return UniformDistribution(value.start, value.end)

    raise NotImplementedError(
        "{} is not supported by Optuna sweeper.".format(override))
Ejemplo n.º 4
0
def test_search_space_transform_untransform_params() -> None:
    search_space = {
        "x0": DiscreteUniformDistribution(0, 1, q=0.2),
        "x1": CategoricalDistribution(["foo", "bar", "baz", "qux"]),
        "x2": IntLogUniformDistribution(1, 10),
        "x3": CategoricalDistribution(["quux", "quuz"]),
        "x4": UniformDistribution(2, 3),
        "x5": LogUniformDistribution(1, 10),
        "x6": IntUniformDistribution(2, 4),
        "x7": CategoricalDistribution(["corge"]),
    }

    params = {
        "x0": 0.2,
        "x1": "qux",
        "x2": 1,
        "x3": "quux",
        "x4": 2.0,
        "x5": 1.0,
        "x6": 2,
        "x7": "corge",
    }

    trans = _SearchSpaceTransform(search_space)
    trans_params = trans.transform(params)
    untrans_params = trans.untransform(trans_params)

    for name in params.keys():
        assert untrans_params[name] == params[name]
Ejemplo n.º 5
0
def test_plot_contour_mixture_category_types() -> None:
    study = create_study()
    study.add_trial(
        create_trial(
            value=0.0,
            params={
                "param_a": None,
                "param_b": 101
            },
            distributions={
                "param_a": CategoricalDistribution([None, "100"]),
                "param_b": CategoricalDistribution([101, 102.0]),
            },
        ))
    study.add_trial(
        create_trial(
            value=0.5,
            params={
                "param_a": "100",
                "param_b": 102.0
            },
            distributions={
                "param_a": CategoricalDistribution([None, "100"]),
                "param_b": CategoricalDistribution([101, 102.0]),
            },
        ))
    figure = plot_contour(study)
    if version.parse(plotly.__version__) >= version.parse("4.12.0"):
        assert figure.layout["xaxis"]["range"] == (-0.05, 1.05)
        assert figure.layout["yaxis"]["range"] == (-0.05, 1.05)
    else:
        assert figure.layout["xaxis"]["range"] == ("100", "None")
        assert figure.layout["yaxis"]["range"] == ("101", "102.0")
    assert figure.layout["xaxis"]["type"] == "category"
    assert figure.layout["yaxis"]["type"] == "category"
Ejemplo n.º 6
0
def test_plot_contour_mixture_category_types() -> None:

    study = create_study()
    study.add_trial(
        create_trial(
            value=0.0,
            params={
                "param_a": None,
                "param_b": 101
            },
            distributions={
                "param_a": CategoricalDistribution([None, "100"]),
                "param_b": CategoricalDistribution([101, 102.0]),
            },
        ))
    study.add_trial(
        create_trial(
            value=0.5,
            params={
                "param_a": "100",
                "param_b": 102.0
            },
            distributions={
                "param_a": CategoricalDistribution([None, "100"]),
                "param_b": CategoricalDistribution([101, 102.0]),
            },
        ))

    figure = plot_contour(study)
    assert figure.get_xlim() == (-0.05, 1.05)
    assert figure.get_ylim() == (100.95, 102.05)
Ejemplo n.º 7
0
def test_plot_parallel_coordinate_categorical_params() -> None:
    # Test with categorical params that cannot be converted to numeral.
    study_categorical_params = create_study()
    study_categorical_params.add_trial(
        create_trial(
            value=0.0,
            params={
                "category_a": "preferred",
                "category_b": "net"
            },
            distributions={
                "category_a": CategoricalDistribution(("preferred", "opt")),
                "category_b": CategoricalDistribution(("net", "una")),
            },
        ))
    study_categorical_params.add_trial(
        create_trial(
            value=2.0,
            params={
                "category_a": "opt",
                "category_b": "una"
            },
            distributions={
                "category_a": CategoricalDistribution(("preferred", "opt")),
                "category_b": CategoricalDistribution(("net", "una")),
            },
        ))
    figure = plot_parallel_coordinate(study_categorical_params)
    assert len(figure.get_lines()) == 0
Ejemplo n.º 8
0
def test_conditional_sample_independent(sampler_class: Callable[[], BaseSampler]) -> None:
    # This test case reproduces the error reported in #2734.
    # See https://github.com/optuna/optuna/pull/2734#issuecomment-857649769.

    study = optuna.study.create_study(sampler=sampler_class())
    categorical_distribution = CategoricalDistribution(choices=["x", "y"])
    dependent_distribution = CategoricalDistribution(choices=["a", "b"])

    study.add_trial(
        optuna.create_trial(
            params={"category": "x", "x": "a"},
            distributions={"category": categorical_distribution, "x": dependent_distribution},
            value=0.1,
        )
    )

    study.add_trial(
        optuna.create_trial(
            params={"category": "y", "y": "b"},
            distributions={"category": categorical_distribution, "y": dependent_distribution},
            value=0.1,
        )
    )

    _trial = _create_new_trial(study)
    category = study.sampler.sample_independent(
        study, _trial, "category", categorical_distribution
    )
    assert category in ["x", "y"]
    value = study.sampler.sample_independent(study, _trial, category, dependent_distribution)
    assert value in ["a", "b"]
Ejemplo n.º 9
0
def test_search_space_transform_untransform_params() -> None:
    search_space = {
        "x0": CategoricalDistribution(["corge"]),
        "x1": CategoricalDistribution(["foo", "bar", "baz", "qux"]),
        "x2": CategoricalDistribution(["quux", "quuz"]),
        "x3": FloatDistribution(2, 3),
        "x4": FloatDistribution(-2, 2),
        "x5": FloatDistribution(1, 10, log=True),
        "x6": FloatDistribution(1, 1, log=True),
        "x7": FloatDistribution(0, 1, step=0.2),
        "x8": IntDistribution(2, 4),
        "x9": IntDistribution(1, 10, log=True),
        "x10": IntDistribution(1, 9, step=2),
    }

    params = {
        "x0": "corge",
        "x1": "qux",
        "x2": "quux",
        "x3": 2.0,
        "x4": -2,
        "x5": 1.0,
        "x6": 1.0,
        "x7": 0.2,
        "x8": 2,
        "x9": 1,
        "x10": 3,
    }

    trans = _SearchSpaceTransform(search_space)
    trans_params = trans.transform(params)
    untrans_params = trans.untransform(trans_params)

    for name in params.keys():
        assert untrans_params[name] == params[name]
Ejemplo n.º 10
0
def _create_study_with_log_scale_and_str_category_3d() -> Study:
    study = create_study()
    distributions = {
        "param_a": FloatDistribution(1e-7, 1e-2, log=True),
        "param_b": CategoricalDistribution(["100", "101"]),
        "param_c": CategoricalDistribution(["one", "two"]),
    }
    study.add_trial(
        create_trial(
            value=0.0,
            params={
                "param_a": 1e-6,
                "param_b": "101",
                "param_c": "one"
            },
            distributions=distributions,
        ))
    study.add_trial(
        create_trial(
            value=1.0,
            params={
                "param_a": 1e-5,
                "param_b": "100",
                "param_c": "two"
            },
            distributions=distributions,
        ))
    return study
def test_plot_parallel_coordinate_categorical_params() -> None:
    # Test with categorical params that cannot be converted to numeral.
    study_categorical_params = create_study()
    distributions: Dict[str, BaseDistribution] = {
        "category_a": CategoricalDistribution(("preferred", "opt")),
        "category_b": CategoricalDistribution(("net", "una")),
    }
    study_categorical_params.add_trial(
        create_trial(
            value=0.0,
            params={"category_a": "preferred", "category_b": "net"},
            distributions=distributions,
        )
    )
    study_categorical_params.add_trial(
        create_trial(
            value=2.0,
            params={"category_a": "opt", "category_b": "una"},
            distributions=distributions,
        )
    )
    figure = plot_parallel_coordinate(study_categorical_params)
    assert len(figure.data[0]["dimensions"]) == 3
    assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
    assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
    assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0)
    assert figure.data[0]["dimensions"][1]["label"] == "category_a"
    assert figure.data[0]["dimensions"][1]["range"] == (0, 1)
    assert figure.data[0]["dimensions"][1]["values"] == (0, 1)
    assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", "opt")
    assert figure.data[0]["dimensions"][2]["label"] == "category_b"
    assert figure.data[0]["dimensions"][2]["range"] == (0, 1)
    assert figure.data[0]["dimensions"][2]["values"] == (0, 1)
    assert figure.data[0]["dimensions"][2]["ticktext"] == ("net", "una")
Ejemplo n.º 12
0
def _create_study_with_numeric_categorical_params() -> Study:
    study_categorical_params = create_study()
    distributions: Dict[str, BaseDistribution] = {
        "category_a": CategoricalDistribution((1, 2)),
        "category_b": CategoricalDistribution((10, 20, 30)),
    }
    study_categorical_params.add_trial(
        create_trial(
            value=0.0,
            params={
                "category_a": 2,
                "category_b": 20
            },
            distributions=distributions,
        ))
    study_categorical_params.add_trial(
        create_trial(
            value=1.0,
            params={
                "category_a": 1,
                "category_b": 30
            },
            distributions=distributions,
        ))
    study_categorical_params.add_trial(
        create_trial(
            value=2.0,
            params={
                "category_a": 2,
                "category_b": 10
            },
            distributions=distributions,
        ))
    return study_categorical_params
Ejemplo n.º 13
0
def _create_study_with_categorical_params() -> Study:
    study_categorical_params = create_study()
    distributions: Dict[str, BaseDistribution] = {
        "category_a": CategoricalDistribution(("preferred", "opt")),
        "category_b": CategoricalDistribution(("net", "una")),
    }
    study_categorical_params.add_trial(
        create_trial(
            value=0.0,
            params={
                "category_a": "preferred",
                "category_b": "net"
            },
            distributions=distributions,
        ))
    study_categorical_params.add_trial(
        create_trial(
            value=2.0,
            params={
                "category_a": "opt",
                "category_b": "una"
            },
            distributions=distributions,
        ))
    return study_categorical_params
Ejemplo n.º 14
0
def test_plot_parallel_coordinate_categorical_params() -> None:
    # Test with categorical params that cannot be converted to numeral.
    # TODO(ytknzw): Add more specific assertion with the test case.
    study_categorical_params = create_study()
    study_categorical_params.add_trial(
        create_trial(
            value=0.0,
            params={
                "category_a": "preferred",
                "category_b": "net"
            },
            distributions={
                "category_a": CategoricalDistribution(("preferred", "opt")),
                "category_b": CategoricalDistribution(("net", "una")),
            },
        ))
    study_categorical_params.add_trial(
        create_trial(
            value=2.0,
            params={
                "category_a": "opt",
                "category_b": "una"
            },
            distributions={
                "category_a": CategoricalDistribution(("preferred", "opt")),
                "category_b": CategoricalDistribution(("net", "una")),
            },
        ))
    figure = plot_parallel_coordinate(study_categorical_params)
    assert figure.has_data()
Ejemplo n.º 15
0
def test_set_and_get_trial_param(storage_init_func):
    # type: (Callable[[], BaseStorage]) -> None

    storage = storage_init_func()

    # Setup test across multiple studies and trials.
    study_id = storage.create_new_study()
    trial_id_1 = storage.create_new_trial(study_id)
    trial_id_2 = storage.create_new_trial(study_id)
    trial_id_3 = storage.create_new_trial(storage.create_new_study())

    # Setup Distributions.
    distribution_x = UniformDistribution(low=1.0, high=2.0)
    distribution_y_1 = CategoricalDistribution(choices=('Shibuya', 'Ebisu',
                                                        'Meguro'))
    distribution_y_2 = CategoricalDistribution(choices=('Shibuya', 'Shinsen'))
    distribution_z = LogUniformDistribution(low=1.0, high=100.0)

    # Test trial_1: setting new params.
    assert storage.set_trial_param(trial_id_1, 'x', 0.5, distribution_x)
    assert storage.set_trial_param(trial_id_1, 'y', 2, distribution_y_1)

    # Test trial_1: getting params.
    assert storage.get_trial_param(trial_id_1, 'x') == 0.5
    assert storage.get_trial_param(trial_id_1, 'y') == 2
    # Test trial_1: checking all params and external repr.
    assert storage.get_trial(trial_id_1).params == {'x': 0.5, 'y': 'Meguro'}
    # Test trial_1: setting existing name.
    assert not storage.set_trial_param(trial_id_1, 'x', 0.6, distribution_x)

    # Setup trial_2: setting new params (to the same study as trial_1).
    assert storage.set_trial_param(trial_id_2, 'x', 0.3, distribution_x)
    assert storage.set_trial_param(trial_id_2, 'z', 0.1, distribution_z)

    # Test trial_2: getting params.
    assert storage.get_trial_param(trial_id_2, 'x') == 0.3
    assert storage.get_trial_param(trial_id_2, 'z') == 0.1

    # Test trial_2: checking all params and external repr.
    assert storage.get_trial(trial_id_2).params == {'x': 0.3, 'z': 0.1}
    # Test trial_2: setting different distribution.
    with pytest.raises(ValueError):
        storage.set_trial_param(trial_id_2, 'x', 0.5, distribution_z)
    # Test trial_2: setting CategoricalDistribution in different order.
    with pytest.raises(ValueError):
        storage.set_trial_param(
            trial_id_2, 'y', 2,
            CategoricalDistribution(choices=('Meguro', 'Shibuya', 'Ebisu')))

    # Setup trial_3: setting new params (to different study from trial_1).
    if isinstance(storage, InMemoryStorage):
        with pytest.raises(ValueError):
            # InMemoryStorage shares the same study if create_new_study is additionally invoked.
            # Thus, the following line should fail due to distribution incompatibility.
            storage.set_trial_param(trial_id_3, 'y', 1, distribution_y_2)
    else:
        assert storage.set_trial_param(trial_id_3, 'y', 1, distribution_y_2)
        assert storage.get_trial_param(trial_id_3, 'y') == 1
        assert storage.get_trial(trial_id_3).params == {'y': 'Shinsen'}
Ejemplo n.º 16
0
def test_set_and_get_trial_param(storage_init_func):
    # type: (Callable[[], BaseStorage]) -> None

    storage = storage_init_func()

    # Setup test across multiple studies and trials.
    study_id = storage.create_new_study()
    trial_id_1 = storage.create_new_trial(study_id)
    trial_id_2 = storage.create_new_trial(study_id)
    trial_id_3 = storage.create_new_trial(storage.create_new_study())

    # Setup Distributions.
    distribution_x = UniformDistribution(low=1.0, high=2.0)
    distribution_y_1 = CategoricalDistribution(choices=("Shibuya", "Ebisu",
                                                        "Meguro"))
    distribution_y_2 = CategoricalDistribution(choices=("Shibuya", "Shinsen"))
    distribution_z = LogUniformDistribution(low=1.0, high=100.0)

    # Test trial_1: setting new params.
    assert storage.set_trial_param(trial_id_1, "x", 0.5, distribution_x)
    assert storage.set_trial_param(trial_id_1, "y", 2, distribution_y_1)

    # Test trial_1: getting params.
    assert storage.get_trial_param(trial_id_1, "x") == 0.5
    assert storage.get_trial_param(trial_id_1, "y") == 2
    # Test trial_1: checking all params and external repr.
    assert storage.get_trial(trial_id_1).params == {"x": 0.5, "y": "Meguro"}
    # Test trial_1: setting existing name.
    assert not storage.set_trial_param(trial_id_1, "x", 0.6, distribution_x)

    # Setup trial_2: setting new params (to the same study as trial_1).
    assert storage.set_trial_param(trial_id_2, "x", 0.3, distribution_x)
    assert storage.set_trial_param(trial_id_2, "z", 0.1, distribution_z)

    # Test trial_2: getting params.
    assert storage.get_trial_param(trial_id_2, "x") == 0.3
    assert storage.get_trial_param(trial_id_2, "z") == 0.1

    # Test trial_2: checking all params and external repr.
    assert storage.get_trial(trial_id_2).params == {"x": 0.3, "z": 0.1}
    # Test trial_2: setting different distribution.
    with pytest.raises(ValueError):
        storage.set_trial_param(trial_id_2, "x", 0.5, distribution_z)
    # Test trial_2: setting CategoricalDistribution in different order.
    with pytest.raises(ValueError):
        storage.set_trial_param(
            trial_id_2, "y", 2,
            CategoricalDistribution(choices=("Meguro", "Shibuya", "Ebisu")))

    # Setup trial_3: setting new params (to different study from trial_1).
    if isinstance(storage, InMemoryStorage):
        pass
    else:
        assert storage.set_trial_param(trial_id_3, "y", 1, distribution_y_2)
        assert storage.get_trial_param(trial_id_3, "y") == 1
        assert storage.get_trial(trial_id_3).params == {"y": "Shinsen"}
Ejemplo n.º 17
0
def _create_study_with_log_scale_and_str_and_numeric_category() -> Study:
    study_multi_distro_params = create_study()
    distributions: Dict[str, BaseDistribution] = {
        "param_a": CategoricalDistribution(("preferred", "opt")),
        "param_b": CategoricalDistribution((1, 2, 10)),
        "param_c": FloatDistribution(1, 1000, log=True),
        "param_d": CategoricalDistribution((1, -1, 2)),
    }
    study_multi_distro_params.add_trial(
        create_trial(
            value=0.0,
            params={
                "param_a": "preferred",
                "param_b": 2,
                "param_c": 30,
                "param_d": 2
            },
            distributions=distributions,
        ))

    study_multi_distro_params.add_trial(
        create_trial(
            value=1.0,
            params={
                "param_a": "opt",
                "param_b": 1,
                "param_c": 200,
                "param_d": 2
            },
            distributions=distributions,
        ))

    study_multi_distro_params.add_trial(
        create_trial(
            value=2.0,
            params={
                "param_a": "preferred",
                "param_b": 10,
                "param_c": 10,
                "param_d": 1
            },
            distributions=distributions,
        ))

    study_multi_distro_params.add_trial(
        create_trial(
            value=3.0,
            params={
                "param_a": "opt",
                "param_b": 2,
                "param_c": 10,
                "param_d": -1
            },
            distributions=distributions,
        ))
    return study_multi_distro_params
Ejemplo n.º 18
0
def test_plot_parallel_coordinate_unique_hyper_param() -> None:
    # Test case when one unique value is suggested during the optimization.

    study_categorical_params = create_study()
    study_categorical_params.add_trial(
        create_trial(
            value=0.0,
            params={
                "category_a": "preferred",
                "param_b": 30
            },
            distributions={
                "category_a": CategoricalDistribution(("preferred", "opt")),
                "param_b": FloatDistribution(1, 1000, log=True),
            },
        ))

    # Both hyperparameters contain unique values.
    figure = plot_parallel_coordinate(study_categorical_params)
    assert len(figure.data[0]["dimensions"]) == 3
    assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
    assert figure.data[0]["dimensions"][0]["range"] == (0.0, 0.0)
    assert figure.data[0]["dimensions"][0]["values"] == (0.0, )
    assert figure.data[0]["dimensions"][1]["label"] == "category_a"
    assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
    assert figure.data[0]["dimensions"][1]["values"] == (0.0, )
    assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
    assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
    assert figure.data[0]["dimensions"][2]["label"] == "param_b"
    assert figure.data[0]["dimensions"][2]["range"] == (math.log10(30),
                                                        math.log10(30))
    assert figure.data[0]["dimensions"][2]["values"] == (math.log10(30), )
    assert figure.data[0]["dimensions"][2]["ticktext"] == ("30", )
    assert figure.data[0]["dimensions"][2]["tickvals"] == (math.log10(30), )

    study_categorical_params.add_trial(
        create_trial(
            value=2.0,
            params={
                "category_a": "preferred",
                "param_b": 20
            },
            distributions={
                "category_a": CategoricalDistribution(("preferred", "opt")),
                "param_b": FloatDistribution(1, 1000, log=True),
            },
        ))

    # Still "category_a" contains unique suggested value during the optimization.
    figure = plot_parallel_coordinate(study_categorical_params)
    assert len(figure.data[0]["dimensions"]) == 3
    assert figure.data[0]["dimensions"][1]["label"] == "category_a"
    assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
    assert figure.data[0]["dimensions"][1]["values"] == (0.0, 0.0)
    assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
    assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
Ejemplo n.º 19
0
def _generate_trial(generator: random.Random) -> FrozenTrial:
    example_params = {
        "paramA": (generator.uniform(0, 1), UniformDistribution(0, 1)),
        "paramB": (generator.uniform(1, 2), LogUniformDistribution(1, 2)),
        "paramC": (
            generator.choice(["CatA", "CatB", "CatC"]),
            CategoricalDistribution(("CatA", "CatB", "CatC")),
        ),
        "paramD": (generator.uniform(-3, 0), UniformDistribution(-3, 0)),
        "paramE":
        (generator.choice([0.1, 0.2]), CategoricalDistribution((0.1, 0.2))),
    }
    example_attrs = {
        "attrA": "valueA",
        "attrB": 1,
        "attrC": None,
        "attrD": {
            "baseline_score": 0.001,
            "tags": ["image", "classification"]
        },
    }
    state = generator.choice(ALL_STATES)
    params = {}
    distributions = {}
    user_attrs = {}
    system_attrs = {}
    intermediate_values = {}
    for key, (value, dist) in example_params.items():
        if generator.choice([True, False]):
            params[key] = value
            distributions[key] = dist
    for key, value in example_attrs.items():
        if generator.choice([True, False]):
            user_attrs["usr_" + key] = value
        if generator.choice([True, False]):
            system_attrs["sys_" + key] = value
    for i in range(generator.randint(4, 10)):
        if generator.choice([True, False]):
            intermediate_values[i] = generator.uniform(-10, 10)
    return FrozenTrial(
        number=0,  # dummy
        state=state,
        value=generator.uniform(-10, 10),
        datetime_start=datetime.now(),
        datetime_complete=datetime.now() if state.is_finished() else None,
        params=params,
        distributions=distributions,
        user_attrs=user_attrs,
        system_attrs=system_attrs,
        intermediate_values=intermediate_values,
        trial_id=0,  # dummy
    )
Ejemplo n.º 20
0
def test_plot_contour_log_scale_and_str_category() -> None:

    # If the search space has three parameters, plot_contour generates nine plots.
    study = create_study()
    study.add_trial(
        create_trial(
            value=0.0,
            params={
                "param_a": 1e-6,
                "param_b": "100",
                "param_c": "one"
            },
            distributions={
                "param_a": FloatDistribution(1e-7, 1e-2, log=True),
                "param_b": CategoricalDistribution(["100", "101"]),
                "param_c": CategoricalDistribution(["one", "two"]),
            },
        ))
    study.add_trial(
        create_trial(
            value=1.0,
            params={
                "param_a": 1e-5,
                "param_b": "101",
                "param_c": "two"
            },
            distributions={
                "param_a": FloatDistribution(1e-7, 1e-2, log=True),
                "param_b": CategoricalDistribution(["100", "101"]),
                "param_c": CategoricalDistribution(["one", "two"]),
            },
        ))

    figure = plot_contour(study)
    subplots = [plot for plot in figure.flatten() if plot.has_data()]
    expected = {
        "param_a": [1e-6, 1e-5],
        "param_b": [0.0, 1.0],
        "param_c": [0.0, 1.0]
    }
    ranges = itertools.permutations(expected.keys(), 2)

    for plot, (yrange, xrange) in zip(subplots, ranges):
        # Take 5% axis padding into account.
        np.testing.assert_allclose(plot.get_xlim(),
                                   expected[xrange],
                                   atol=5e-2)
        np.testing.assert_allclose(plot.get_ylim(),
                                   expected[yrange],
                                   atol=5e-2)
    plt.savefig(BytesIO())
Ejemplo n.º 21
0
def test_suggest_categorical() -> None:

    # Integer categories.
    trial = FrozenTrial(
        number=0,
        trial_id=0,
        state=TrialState.COMPLETE,
        value=0.2,
        datetime_start=datetime.datetime.now(),
        datetime_complete=datetime.datetime.now(),
        params={"x": 1},
        distributions={"x": CategoricalDistribution((0, 1, 2, 3))},
        user_attrs={},
        system_attrs={},
        intermediate_values={},
    )
    assert trial.suggest_categorical("x", (0, 1, 2, 3)) == 1

    with pytest.raises(ValueError):
        trial.suggest_categorical("y", [0, 1, 2, 3])

    # String categories.
    trial = FrozenTrial(
        number=0,
        trial_id=0,
        state=TrialState.COMPLETE,
        value=0.2,
        datetime_start=datetime.datetime.now(),
        datetime_complete=datetime.datetime.now(),
        params={"x": "baz"},
        distributions={"x": CategoricalDistribution(("foo", "bar", "baz"))},
        user_attrs={},
        system_attrs={},
        intermediate_values={},
    )
    assert trial.suggest_categorical("x", ("foo", "bar", "baz")) == "baz"

    # Unknown parameter.
    with pytest.raises(ValueError):
        trial.suggest_categorical("y", ["foo", "bar", "baz"])

    # Not in choices.
    with pytest.raises(ValueError):
        trial.suggest_categorical("x", ["foo", "bar"])

    # Unknown parameter and bad category type.
    with pytest.warns(UserWarning):
        with pytest.raises(
                ValueError
        ):  # Must come after `pytest.warns` to catch failures.
            trial.suggest_categorical("x", [{"foo": "bar"}])  # type: ignore
Ejemplo n.º 22
0
def test_sample_single_distribution(
        sampler_class: Callable[[], BaseSampler]) -> None:

    relative_search_space = {
        "a": UniformDistribution(low=1.0, high=1.0),
        "b": LogUniformDistribution(low=1.0, high=1.0),
        "c": DiscreteUniformDistribution(low=1.0, high=1.0, q=1.0),
        "d": IntUniformDistribution(low=1, high=1),
        "e": IntLogUniformDistribution(low=1, high=1),
        "f": CategoricalDistribution([1]),
        "g": FloatDistribution(low=1.0, high=1.0),
        "h": FloatDistribution(low=1.0, high=1.0, log=True),
        "i": FloatDistribution(low=1.0, high=1.0, step=1.0),
        "j": IntDistribution(low=1, high=1),
        "k": IntDistribution(low=1, high=1, log=True),
    }

    with warnings.catch_warnings():
        warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
        sampler = sampler_class()
    study = optuna.study.create_study(sampler=sampler)

    # We need to test the construction of the model, so we should set `n_trials >= 2`.
    for _ in range(2):
        trial = study.ask(fixed_distributions=relative_search_space)
        study.tell(trial, 1.0)
        for param_name in relative_search_space.keys():
            assert trial.params[param_name] == 1
Ejemplo n.º 23
0
def test_sample_relative() -> None:

    relative_search_space: Dict[str, BaseDistribution] = {
        "a": UniformDistribution(low=0, high=5),
        "b": CategoricalDistribution(choices=("foo", "bar", "baz")),
        "c": IntUniformDistribution(low=20, high=50),  # Not exist in `relative_params`.
    }
    relative_params = {
        "a": 3.2,
        "b": "baz",
    }
    unknown_param_value = 30

    sampler = FixedSampler(  # type: ignore
        relative_search_space, relative_params, unknown_param_value
    )
    study = optuna.study.create_study(sampler=sampler)

    def objective(trial: Trial) -> float:

        # Predefined parameters are sampled by `sample_relative()` method.
        assert trial.suggest_uniform("a", 0, 5) == 3.2
        assert trial.suggest_categorical("b", ["foo", "bar", "baz"]) == "baz"

        # Other parameters are sampled by `sample_independent()` method.
        assert trial.suggest_int("c", 20, 50) == unknown_param_value
        assert trial.suggest_loguniform("d", 1, 100) == unknown_param_value
        assert trial.suggest_uniform("e", 20, 40) == unknown_param_value

        return 0.0

    study.optimize(objective, n_trials=10, catch=())
    for trial in study.trials:
        assert trial.params == {"a": 3.2, "b": "baz", "c": 30, "d": 30, "e": 30}
Ejemplo n.º 24
0
def test_set_trial_param_to_check_distribution_json():
    # type: () -> None

    example_distributions = {
        "x": UniformDistribution(low=1.0, high=2.0),
        "y": CategoricalDistribution(choices=("Otemachi", "Tokyo", "Ginza")),
    }  # type: Dict[str, BaseDistribution]

    storage = create_test_storage()
    session = storage.scoped_session()
    study_id = storage.create_new_study()

    trial_id = storage.create_new_trial(study_id)
    storage.set_trial_param(trial_id, "x", 1.5, example_distributions["x"])
    storage.set_trial_param(trial_id, "y", 2, example_distributions["y"])

    # test setting new name
    result_1 = session.query(TrialParamModel).filter(
        TrialParamModel.param_name == "x").one()
    assert json_to_distribution(
        result_1.distribution_json) == example_distributions["x"]

    result_2 = session.query(TrialParamModel).filter(
        TrialParamModel.param_name == "y").one()
    assert json_to_distribution(
        result_2.distribution_json) == example_distributions["y"]
Ejemplo n.º 25
0
def test_distributions(storage_init_func):
    # type: (Callable[[], storages.BaseStorage]) -> None

    def objective(trial):
        # type: (Trial) -> float

        trial.suggest_uniform("a", 0, 10)
        trial.suggest_loguniform("b", 0.1, 10)
        trial.suggest_discrete_uniform("c", 0, 10, 1)
        trial.suggest_int("d", 0, 10)
        trial.suggest_categorical("e", ["foo", "bar", "baz"])
        trial.suggest_int("f", 1, 10, log=True)

        return 1.0

    study = create_study(storage_init_func())
    study.optimize(objective, n_trials=1)

    assert study.best_trial.distributions == {
        "a": UniformDistribution(low=0, high=10),
        "b": LogUniformDistribution(low=0.1, high=10),
        "c": DiscreteUniformDistribution(low=0, high=10, q=1),
        "d": IntUniformDistribution(low=0, high=10),
        "e": CategoricalDistribution(choices=("foo", "bar", "baz")),
        "f": IntLogUniformDistribution(low=1, high=10),
    }
Ejemplo n.º 26
0
def test_set_trial_param_to_check_distribution_json():
    # type: () -> None

    example_distributions = {
        'x': UniformDistribution(low=1., high=2.),
        'y': CategoricalDistribution(choices=('Otemachi', 'Tokyo', 'Ginza'))
    }  # type: Dict[str, BaseDistribution]

    storage = create_test_storage()
    session = storage.scoped_session()
    study_id = storage.create_new_study_id()

    trial_id = storage.create_new_trial_id(study_id)
    storage.set_trial_param(trial_id, 'x', 1.5, example_distributions['x'])
    storage.set_trial_param(trial_id, 'y', 2, example_distributions['y'])

    # test setting new name
    result_1 = session.query(TrialParamModel). \
        filter(TrialParamModel.param_name == 'x').one()
    assert json_to_distribution(
        result_1.distribution_json) == example_distributions['x']

    result_2 = session.query(TrialParamModel). \
        filter(TrialParamModel.param_name == 'y').one()
    assert json_to_distribution(
        result_2.distribution_json) == example_distributions['y']
Ejemplo n.º 27
0
def test_shap_importance_evaluator_with_infinite(inf_value: float) -> None:
    # The test ensures that trials with infinite values are ignored to calculate importance scores.
    n_trial = 10
    seed = 13

    # Importance scores are calculated without a trial with an inf value.
    study = create_study(sampler=RandomSampler(seed=seed))
    study.optimize(objective, n_trials=n_trial)

    evaluator = ShapleyImportanceEvaluator(seed=seed)
    param_importance_without_inf = evaluator.evaluate(study)

    # A trial with an inf value is added into the study manually.
    study.add_trial(
        create_trial(
            value=inf_value,
            params={"x1": 1.0, "x2": 1.0, "x3": 3.0, "x4": 0.1},
            distributions={
                "x1": FloatDistribution(low=0.1, high=3),
                "x2": FloatDistribution(low=0.1, high=3, log=True),
                "x3": IntDistribution(low=2, high=4, log=True),
                "x4": CategoricalDistribution([0.1, 1, 10]),
            },
        )
    )
    # Importance scores are calculated with a trial with an inf value.
    param_importance_with_inf = evaluator.evaluate(study)

    # Obtained importance scores should be the same between with inf and without inf,
    # because the last trial whose objective value is an inf is ignored.
    assert param_importance_with_inf == param_importance_without_inf
Ejemplo n.º 28
0
def create_optuna_distribution_from_config(
        config: MutableMapping[str, Any]) -> BaseDistribution:
    kwargs = dict(config)
    if isinstance(config["type"], str):
        kwargs["type"] = DistributionType[config["type"]]
    param = DistributionConfig(**kwargs)
    if param.type == DistributionType.categorical:
        assert param.choices is not None
        return CategoricalDistribution(param.choices)
    if param.type == DistributionType.int:
        assert param.low is not None
        assert param.high is not None
        if param.log:
            return IntLogUniformDistribution(int(param.low), int(param.high))
        step = int(param.step) if param.step is not None else 1
        return IntUniformDistribution(int(param.low),
                                      int(param.high),
                                      step=step)
    if param.type == DistributionType.float:
        assert param.low is not None
        assert param.high is not None
        if param.log:
            return LogUniformDistribution(param.low, param.high)
        if param.step is not None:
            return DiscreteUniformDistribution(param.low, param.high,
                                               param.step)
        return UniformDistribution(param.low, param.high)
    raise NotImplementedError(
        f"{param.type} is not supported by Optuna sweeper.")
Ejemplo n.º 29
0
def test_sample_relative():
    # type: () -> None

    relative_search_space = {
        'a': UniformDistribution(low=0, high=5),
        'b': CategoricalDistribution(choices=('foo', 'bar', 'baz')),
        'c': IntUniformDistribution(low=20, high=50),  # Not exist in `relative_params`.
    }
    relative_params = {
        'a': 3.2,
        'b': 'baz',
    }
    unknown_param_value = 30

    sampler = FixedSampler(  # type: ignore
        relative_search_space, relative_params, unknown_param_value)
    study = optuna.study.create_study(sampler=sampler)

    def objective(trial):
        # type: (Trial) -> float

        # Predefined parameters are sampled by `sample_relative()` method.
        assert trial.suggest_uniform('a', 0, 5) == 3.2
        assert trial.suggest_categorical('b', ['foo', 'bar', 'baz']) == 'baz'

        # Other parameters are sampled by `sample_independent()` method.
        assert trial.suggest_int('c', 20, 50) == unknown_param_value
        assert trial.suggest_loguniform('d', 1, 100) == unknown_param_value
        assert trial.suggest_uniform('e', 20, 40) == unknown_param_value

        return 0.0

    study.optimize(objective, n_trials=10, catch=())
    for trial in study.trials:
        assert trial.params == {'a': 3.2, 'b': 'baz', 'c': 30, 'd': 30, 'e': 30}
Ejemplo n.º 30
0
def create_optuna_distribution_from_override(override: Override) -> Any:
    value = override.value()
    if not override.is_sweep_override():
        return value

    choices: List[CategoricalChoiceType] = []
    if override.is_choice_sweep():
        assert isinstance(value, ChoiceSweep)
        for x in override.sweep_iterator(transformer=Transformer.encode):
            assert isinstance(
                x, (str, int, float, bool)
            ), f"A choice sweep expects str, int, float, or bool type. Got {type(x)}."
            choices.append(x)
        return CategoricalDistribution(choices)

    if override.is_range_sweep():
        assert isinstance(value, RangeSweep)
        assert value.start is not None
        assert value.stop is not None
        if value.shuffle:
            for x in override.sweep_iterator(transformer=Transformer.encode):
                assert isinstance(
                    x, (str, int, float, bool)
                ), f"A choice sweep expects str, int, float, or bool type. Got {type(x)}."
                choices.append(x)
            return CategoricalDistribution(choices)
        return IntUniformDistribution(int(value.start),
                                      int(value.stop),
                                      step=int(value.step))

    if override.is_interval_sweep():
        assert isinstance(value, IntervalSweep)
        assert value.start is not None
        assert value.end is not None
        if "log" in value.tags:
            if isinstance(value.start, int) and isinstance(value.end, int):
                return IntLogUniformDistribution(int(value.start),
                                                 int(value.end))
            return LogUniformDistribution(value.start, value.end)
        else:
            if isinstance(value.start, int) and isinstance(value.end, int):
                return IntUniformDistribution(value.start, value.end)
            return UniformDistribution(value.start, value.end)

    raise NotImplementedError(
        f"{override} is not supported by Optuna sweeper.")