def test_suggest_categorical(trial_type: type) -> None:
# Integer categories.
trial = _create_trial(
trial_type=trial_type,
params={"x": 1},
distributions={"x": CategoricalDistribution((0, 1, 2, 3))},
)
assert trial.suggest_categorical("x", (0, 1, 2, 3)) == 1
with pytest.raises(ValueError):
trial.suggest_categorical("y", [0, 1, 2, 3])
# String categories.
trial = _create_trial(
trial_type=trial_type,
params={"x": "baz"},
distributions={"x": CategoricalDistribution(("foo", "bar", "baz"))},
)
assert trial.suggest_categorical("x", ("foo", "bar", "baz")) == "baz"
# Unknown parameter.
with pytest.raises(ValueError):
trial.suggest_categorical("y", ["foo", "bar", "baz"])
# Not in choices.
with pytest.raises(ValueError):
trial.suggest_categorical("x", ["foo", "bar"])
# Unknown parameter and bad category type.
with pytest.warns(UserWarning):
with pytest.raises(
ValueError
): # Must come after `pytest.warns` to catch failures.
trial.suggest_categorical("x", [{"foo": "bar"}]) # type: ignore
def test_search_space_transform_encoding() -> None:
trans = _SearchSpaceTransform({"x0": IntUniformDistribution(0, 3)})
assert len(trans.column_to_encoded_columns) == 1
numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0]))
numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0]))
trans = _SearchSpaceTransform({"x0": CategoricalDistribution(["foo", "bar", "baz"])})
assert len(trans.column_to_encoded_columns) == 1
numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0, 1, 2]))
numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0, 0, 0]))
trans = _SearchSpaceTransform(
{
"x0": UniformDistribution(0, 3),
"x1": CategoricalDistribution(["foo", "bar", "baz"]),
"x3": DiscreteUniformDistribution(0, 1, q=0.2),
}
)
assert len(trans.column_to_encoded_columns) == 3
numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0]))
numpy.testing.assert_equal(trans.column_to_encoded_columns[1], numpy.array([1, 2, 3]))
numpy.testing.assert_equal(trans.column_to_encoded_columns[2], numpy.array([4]))
numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0, 1, 1, 1, 2]))
def create_optuna_distribution_from_override(override: Override) -> Any:
value = override.value()
if not override.is_sweep_override():
return value
if override.is_choice_sweep():
assert isinstance(value, ChoiceSweep)
choices = [
x for x in override.sweep_iterator(transformer=Transformer.encode)
]
return CategoricalDistribution(choices)
if override.is_range_sweep():
choices = [
x for x in override.sweep_iterator(transformer=Transformer.encode)
]
return CategoricalDistribution(choices)
if override.is_interval_sweep():
assert isinstance(value, IntervalSweep)
if "log" in value.tags:
if "int" in value.tags:
return IntLogUniformDistribution(value.start, value.end)
return LogUniformDistribution(value.start, value.end)
else:
if "int" in value.tags:
return IntUniformDistribution(value.start, value.end)
return UniformDistribution(value.start, value.end)
raise NotImplementedError(
"{} is not supported by Optuna sweeper.".format(override))
def test_search_space_transform_untransform_params() -> None:
search_space = {
"x0": DiscreteUniformDistribution(0, 1, q=0.2),
"x1": CategoricalDistribution(["foo", "bar", "baz", "qux"]),
"x2": IntLogUniformDistribution(1, 10),
"x3": CategoricalDistribution(["quux", "quuz"]),
"x4": UniformDistribution(2, 3),
"x5": LogUniformDistribution(1, 10),
"x6": IntUniformDistribution(2, 4),
"x7": CategoricalDistribution(["corge"]),
}
params = {
"x0": 0.2,
"x1": "qux",
"x2": 1,
"x3": "quux",
"x4": 2.0,
"x5": 1.0,
"x6": 2,
"x7": "corge",
}
trans = _SearchSpaceTransform(search_space)
trans_params = trans.transform(params)
untrans_params = trans.untransform(trans_params)
for name in params.keys():
assert untrans_params[name] == params[name]
def test_plot_contour_mixture_category_types() -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={
"param_a": None,
"param_b": 101
},
distributions={
"param_a": CategoricalDistribution([None, "100"]),
"param_b": CategoricalDistribution([101, 102.0]),
},
))
study.add_trial(
create_trial(
value=0.5,
params={
"param_a": "100",
"param_b": 102.0
},
distributions={
"param_a": CategoricalDistribution([None, "100"]),
"param_b": CategoricalDistribution([101, 102.0]),
},
))
figure = plot_contour(study)
if version.parse(plotly.__version__) >= version.parse("4.12.0"):
assert figure.layout["xaxis"]["range"] == (-0.05, 1.05)
assert figure.layout["yaxis"]["range"] == (-0.05, 1.05)
else:
assert figure.layout["xaxis"]["range"] == ("100", "None")
assert figure.layout["yaxis"]["range"] == ("101", "102.0")
assert figure.layout["xaxis"]["type"] == "category"
assert figure.layout["yaxis"]["type"] == "category"
def test_plot_contour_mixture_category_types() -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={
"param_a": None,
"param_b": 101
},
distributions={
"param_a": CategoricalDistribution([None, "100"]),
"param_b": CategoricalDistribution([101, 102.0]),
},
))
study.add_trial(
create_trial(
value=0.5,
params={
"param_a": "100",
"param_b": 102.0
},
distributions={
"param_a": CategoricalDistribution([None, "100"]),
"param_b": CategoricalDistribution([101, 102.0]),
},
))
figure = plot_contour(study)
assert figure.get_xlim() == (-0.05, 1.05)
assert figure.get_ylim() == (100.95, 102.05)
def test_plot_parallel_coordinate_categorical_params() -> None:
# Test with categorical params that cannot be converted to numeral.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"category_b": "net"
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
))
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "opt",
"category_b": "una"
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
))
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.get_lines()) == 0
def test_conditional_sample_independent(sampler_class: Callable[[], BaseSampler]) -> None:
# This test case reproduces the error reported in #2734.
# See https://github.com/optuna/optuna/pull/2734#issuecomment-857649769.
study = optuna.study.create_study(sampler=sampler_class())
categorical_distribution = CategoricalDistribution(choices=["x", "y"])
dependent_distribution = CategoricalDistribution(choices=["a", "b"])
study.add_trial(
optuna.create_trial(
params={"category": "x", "x": "a"},
distributions={"category": categorical_distribution, "x": dependent_distribution},
value=0.1,
)
)
study.add_trial(
optuna.create_trial(
params={"category": "y", "y": "b"},
distributions={"category": categorical_distribution, "y": dependent_distribution},
value=0.1,
)
)
_trial = _create_new_trial(study)
category = study.sampler.sample_independent(
study, _trial, "category", categorical_distribution
)
assert category in ["x", "y"]
value = study.sampler.sample_independent(study, _trial, category, dependent_distribution)
assert value in ["a", "b"]
def test_search_space_transform_untransform_params() -> None:
search_space = {
"x0": CategoricalDistribution(["corge"]),
"x1": CategoricalDistribution(["foo", "bar", "baz", "qux"]),
"x2": CategoricalDistribution(["quux", "quuz"]),
"x3": FloatDistribution(2, 3),
"x4": FloatDistribution(-2, 2),
"x5": FloatDistribution(1, 10, log=True),
"x6": FloatDistribution(1, 1, log=True),
"x7": FloatDistribution(0, 1, step=0.2),
"x8": IntDistribution(2, 4),
"x9": IntDistribution(1, 10, log=True),
"x10": IntDistribution(1, 9, step=2),
}
params = {
"x0": "corge",
"x1": "qux",
"x2": "quux",
"x3": 2.0,
"x4": -2,
"x5": 1.0,
"x6": 1.0,
"x7": 0.2,
"x8": 2,
"x9": 1,
"x10": 3,
}
trans = _SearchSpaceTransform(search_space)
trans_params = trans.transform(params)
untrans_params = trans.untransform(trans_params)
for name in params.keys():
assert untrans_params[name] == params[name]
def _create_study_with_log_scale_and_str_category_3d() -> Study:
study = create_study()
distributions = {
"param_a": FloatDistribution(1e-7, 1e-2, log=True),
"param_b": CategoricalDistribution(["100", "101"]),
"param_c": CategoricalDistribution(["one", "two"]),
}
study.add_trial(
create_trial(
value=0.0,
params={
"param_a": 1e-6,
"param_b": "101",
"param_c": "one"
},
distributions=distributions,
))
study.add_trial(
create_trial(
value=1.0,
params={
"param_a": 1e-5,
"param_b": "100",
"param_c": "two"
},
distributions=distributions,
))
return study
def test_plot_parallel_coordinate_categorical_params() -> None:
# Test with categorical params that cannot be converted to numeral.
study_categorical_params = create_study()
distributions: Dict[str, BaseDistribution] = {
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
}
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={"category_a": "preferred", "category_b": "net"},
distributions=distributions,
)
)
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={"category_a": "opt", "category_b": "una"},
distributions=distributions,
)
)
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 1)
assert figure.data[0]["dimensions"][1]["values"] == (0, 1)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", "opt")
assert figure.data[0]["dimensions"][2]["label"] == "category_b"
assert figure.data[0]["dimensions"][2]["range"] == (0, 1)
assert figure.data[0]["dimensions"][2]["values"] == (0, 1)
assert figure.data[0]["dimensions"][2]["ticktext"] == ("net", "una")
def _create_study_with_numeric_categorical_params() -> Study:
study_categorical_params = create_study()
distributions: Dict[str, BaseDistribution] = {
"category_a": CategoricalDistribution((1, 2)),
"category_b": CategoricalDistribution((10, 20, 30)),
}
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": 2,
"category_b": 20
},
distributions=distributions,
))
study_categorical_params.add_trial(
create_trial(
value=1.0,
params={
"category_a": 1,
"category_b": 30
},
distributions=distributions,
))
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": 2,
"category_b": 10
},
distributions=distributions,
))
return study_categorical_params
def _create_study_with_categorical_params() -> Study:
study_categorical_params = create_study()
distributions: Dict[str, BaseDistribution] = {
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
}
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"category_b": "net"
},
distributions=distributions,
))
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "opt",
"category_b": "una"
},
distributions=distributions,
))
return study_categorical_params
def test_plot_parallel_coordinate_categorical_params() -> None:
# Test with categorical params that cannot be converted to numeral.
# TODO(ytknzw): Add more specific assertion with the test case.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"category_b": "net"
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
))
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "opt",
"category_b": "una"
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
))
figure = plot_parallel_coordinate(study_categorical_params)
assert figure.has_data()
def test_set_and_get_trial_param(storage_init_func):
# type: (Callable[[], BaseStorage]) -> None
storage = storage_init_func()
# Setup test across multiple studies and trials.
study_id = storage.create_new_study()
trial_id_1 = storage.create_new_trial(study_id)
trial_id_2 = storage.create_new_trial(study_id)
trial_id_3 = storage.create_new_trial(storage.create_new_study())
# Setup Distributions.
distribution_x = UniformDistribution(low=1.0, high=2.0)
distribution_y_1 = CategoricalDistribution(choices=('Shibuya', 'Ebisu',
'Meguro'))
distribution_y_2 = CategoricalDistribution(choices=('Shibuya', 'Shinsen'))
distribution_z = LogUniformDistribution(low=1.0, high=100.0)
# Test trial_1: setting new params.
assert storage.set_trial_param(trial_id_1, 'x', 0.5, distribution_x)
assert storage.set_trial_param(trial_id_1, 'y', 2, distribution_y_1)
# Test trial_1: getting params.
assert storage.get_trial_param(trial_id_1, 'x') == 0.5
assert storage.get_trial_param(trial_id_1, 'y') == 2
# Test trial_1: checking all params and external repr.
assert storage.get_trial(trial_id_1).params == {'x': 0.5, 'y': 'Meguro'}
# Test trial_1: setting existing name.
assert not storage.set_trial_param(trial_id_1, 'x', 0.6, distribution_x)
# Setup trial_2: setting new params (to the same study as trial_1).
assert storage.set_trial_param(trial_id_2, 'x', 0.3, distribution_x)
assert storage.set_trial_param(trial_id_2, 'z', 0.1, distribution_z)
# Test trial_2: getting params.
assert storage.get_trial_param(trial_id_2, 'x') == 0.3
assert storage.get_trial_param(trial_id_2, 'z') == 0.1
# Test trial_2: checking all params and external repr.
assert storage.get_trial(trial_id_2).params == {'x': 0.3, 'z': 0.1}
# Test trial_2: setting different distribution.
with pytest.raises(ValueError):
storage.set_trial_param(trial_id_2, 'x', 0.5, distribution_z)
# Test trial_2: setting CategoricalDistribution in different order.
with pytest.raises(ValueError):
storage.set_trial_param(
trial_id_2, 'y', 2,
CategoricalDistribution(choices=('Meguro', 'Shibuya', 'Ebisu')))
# Setup trial_3: setting new params (to different study from trial_1).
if isinstance(storage, InMemoryStorage):
with pytest.raises(ValueError):
# InMemoryStorage shares the same study if create_new_study is additionally invoked.
# Thus, the following line should fail due to distribution incompatibility.
storage.set_trial_param(trial_id_3, 'y', 1, distribution_y_2)
else:
assert storage.set_trial_param(trial_id_3, 'y', 1, distribution_y_2)
assert storage.get_trial_param(trial_id_3, 'y') == 1
assert storage.get_trial(trial_id_3).params == {'y': 'Shinsen'}
def test_set_and_get_trial_param(storage_init_func):
# type: (Callable[[], BaseStorage]) -> None
storage = storage_init_func()
# Setup test across multiple studies and trials.
study_id = storage.create_new_study()
trial_id_1 = storage.create_new_trial(study_id)
trial_id_2 = storage.create_new_trial(study_id)
trial_id_3 = storage.create_new_trial(storage.create_new_study())
# Setup Distributions.
distribution_x = UniformDistribution(low=1.0, high=2.0)
distribution_y_1 = CategoricalDistribution(choices=("Shibuya", "Ebisu",
"Meguro"))
distribution_y_2 = CategoricalDistribution(choices=("Shibuya", "Shinsen"))
distribution_z = LogUniformDistribution(low=1.0, high=100.0)
# Test trial_1: setting new params.
assert storage.set_trial_param(trial_id_1, "x", 0.5, distribution_x)
assert storage.set_trial_param(trial_id_1, "y", 2, distribution_y_1)
# Test trial_1: getting params.
assert storage.get_trial_param(trial_id_1, "x") == 0.5
assert storage.get_trial_param(trial_id_1, "y") == 2
# Test trial_1: checking all params and external repr.
assert storage.get_trial(trial_id_1).params == {"x": 0.5, "y": "Meguro"}
# Test trial_1: setting existing name.
assert not storage.set_trial_param(trial_id_1, "x", 0.6, distribution_x)
# Setup trial_2: setting new params (to the same study as trial_1).
assert storage.set_trial_param(trial_id_2, "x", 0.3, distribution_x)
assert storage.set_trial_param(trial_id_2, "z", 0.1, distribution_z)
# Test trial_2: getting params.
assert storage.get_trial_param(trial_id_2, "x") == 0.3
assert storage.get_trial_param(trial_id_2, "z") == 0.1
# Test trial_2: checking all params and external repr.
assert storage.get_trial(trial_id_2).params == {"x": 0.3, "z": 0.1}
# Test trial_2: setting different distribution.
with pytest.raises(ValueError):
storage.set_trial_param(trial_id_2, "x", 0.5, distribution_z)
# Test trial_2: setting CategoricalDistribution in different order.
with pytest.raises(ValueError):
storage.set_trial_param(
trial_id_2, "y", 2,
CategoricalDistribution(choices=("Meguro", "Shibuya", "Ebisu")))
# Setup trial_3: setting new params (to different study from trial_1).
if isinstance(storage, InMemoryStorage):
pass
else:
assert storage.set_trial_param(trial_id_3, "y", 1, distribution_y_2)
assert storage.get_trial_param(trial_id_3, "y") == 1
assert storage.get_trial(trial_id_3).params == {"y": "Shinsen"}
def _create_study_with_log_scale_and_str_and_numeric_category() -> Study:
study_multi_distro_params = create_study()
distributions: Dict[str, BaseDistribution] = {
"param_a": CategoricalDistribution(("preferred", "opt")),
"param_b": CategoricalDistribution((1, 2, 10)),
"param_c": FloatDistribution(1, 1000, log=True),
"param_d": CategoricalDistribution((1, -1, 2)),
}
study_multi_distro_params.add_trial(
create_trial(
value=0.0,
params={
"param_a": "preferred",
"param_b": 2,
"param_c": 30,
"param_d": 2
},
distributions=distributions,
))
study_multi_distro_params.add_trial(
create_trial(
value=1.0,
params={
"param_a": "opt",
"param_b": 1,
"param_c": 200,
"param_d": 2
},
distributions=distributions,
))
study_multi_distro_params.add_trial(
create_trial(
value=2.0,
params={
"param_a": "preferred",
"param_b": 10,
"param_c": 10,
"param_d": 1
},
distributions=distributions,
))
study_multi_distro_params.add_trial(
create_trial(
value=3.0,
params={
"param_a": "opt",
"param_b": 2,
"param_c": 10,
"param_d": -1
},
distributions=distributions,
))
return study_multi_distro_params
def test_plot_parallel_coordinate_unique_hyper_param() -> None:
# Test case when one unique value is suggested during the optimization.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"param_b": 30
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": FloatDistribution(1, 1000, log=True),
},
))
# Both hyperparameters contain unique values.
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 0.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, )
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
assert figure.data[0]["dimensions"][1]["values"] == (0.0, )
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (math.log10(30),
math.log10(30))
assert figure.data[0]["dimensions"][2]["values"] == (math.log10(30), )
assert figure.data[0]["dimensions"][2]["ticktext"] == ("30", )
assert figure.data[0]["dimensions"][2]["tickvals"] == (math.log10(30), )
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "preferred",
"param_b": 20
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": FloatDistribution(1, 1000, log=True),
},
))
# Still "category_a" contains unique suggested value during the optimization.
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
assert figure.data[0]["dimensions"][1]["values"] == (0.0, 0.0)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
def _generate_trial(generator: random.Random) -> FrozenTrial:
example_params = {
"paramA": (generator.uniform(0, 1), UniformDistribution(0, 1)),
"paramB": (generator.uniform(1, 2), LogUniformDistribution(1, 2)),
"paramC": (
generator.choice(["CatA", "CatB", "CatC"]),
CategoricalDistribution(("CatA", "CatB", "CatC")),
),
"paramD": (generator.uniform(-3, 0), UniformDistribution(-3, 0)),
"paramE":
(generator.choice([0.1, 0.2]), CategoricalDistribution((0.1, 0.2))),
}
example_attrs = {
"attrA": "valueA",
"attrB": 1,
"attrC": None,
"attrD": {
"baseline_score": 0.001,
"tags": ["image", "classification"]
},
}
state = generator.choice(ALL_STATES)
params = {}
distributions = {}
user_attrs = {}
system_attrs = {}
intermediate_values = {}
for key, (value, dist) in example_params.items():
if generator.choice([True, False]):
params[key] = value
distributions[key] = dist
for key, value in example_attrs.items():
if generator.choice([True, False]):
user_attrs["usr_" + key] = value
if generator.choice([True, False]):
system_attrs["sys_" + key] = value
for i in range(generator.randint(4, 10)):
if generator.choice([True, False]):
intermediate_values[i] = generator.uniform(-10, 10)
return FrozenTrial(
number=0, # dummy
state=state,
value=generator.uniform(-10, 10),
datetime_start=datetime.now(),
datetime_complete=datetime.now() if state.is_finished() else None,
params=params,
distributions=distributions,
user_attrs=user_attrs,
system_attrs=system_attrs,
intermediate_values=intermediate_values,
trial_id=0, # dummy
)
def test_plot_contour_log_scale_and_str_category() -> None:
# If the search space has three parameters, plot_contour generates nine plots.
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={
"param_a": 1e-6,
"param_b": "100",
"param_c": "one"
},
distributions={
"param_a": FloatDistribution(1e-7, 1e-2, log=True),
"param_b": CategoricalDistribution(["100", "101"]),
"param_c": CategoricalDistribution(["one", "two"]),
},
))
study.add_trial(
create_trial(
value=1.0,
params={
"param_a": 1e-5,
"param_b": "101",
"param_c": "two"
},
distributions={
"param_a": FloatDistribution(1e-7, 1e-2, log=True),
"param_b": CategoricalDistribution(["100", "101"]),
"param_c": CategoricalDistribution(["one", "two"]),
},
))
figure = plot_contour(study)
subplots = [plot for plot in figure.flatten() if plot.has_data()]
expected = {
"param_a": [1e-6, 1e-5],
"param_b": [0.0, 1.0],
"param_c": [0.0, 1.0]
}
ranges = itertools.permutations(expected.keys(), 2)
for plot, (yrange, xrange) in zip(subplots, ranges):
# Take 5% axis padding into account.
np.testing.assert_allclose(plot.get_xlim(),
expected[xrange],
atol=5e-2)
np.testing.assert_allclose(plot.get_ylim(),
expected[yrange],
atol=5e-2)
plt.savefig(BytesIO())
def test_suggest_categorical() -> None:
# Integer categories.
trial = FrozenTrial(
number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.datetime.now(),
datetime_complete=datetime.datetime.now(),
params={"x": 1},
distributions={"x": CategoricalDistribution((0, 1, 2, 3))},
user_attrs={},
system_attrs={},
intermediate_values={},
)
assert trial.suggest_categorical("x", (0, 1, 2, 3)) == 1
with pytest.raises(ValueError):
trial.suggest_categorical("y", [0, 1, 2, 3])
# String categories.
trial = FrozenTrial(
number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.datetime.now(),
datetime_complete=datetime.datetime.now(),
params={"x": "baz"},
distributions={"x": CategoricalDistribution(("foo", "bar", "baz"))},
user_attrs={},
system_attrs={},
intermediate_values={},
)
assert trial.suggest_categorical("x", ("foo", "bar", "baz")) == "baz"
# Unknown parameter.
with pytest.raises(ValueError):
trial.suggest_categorical("y", ["foo", "bar", "baz"])
# Not in choices.
with pytest.raises(ValueError):
trial.suggest_categorical("x", ["foo", "bar"])
# Unknown parameter and bad category type.
with pytest.warns(UserWarning):
with pytest.raises(
ValueError
): # Must come after `pytest.warns` to catch failures.
trial.suggest_categorical("x", [{"foo": "bar"}]) # type: ignore
def test_sample_single_distribution(
sampler_class: Callable[[], BaseSampler]) -> None:
relative_search_space = {
"a": UniformDistribution(low=1.0, high=1.0),
"b": LogUniformDistribution(low=1.0, high=1.0),
"c": DiscreteUniformDistribution(low=1.0, high=1.0, q=1.0),
"d": IntUniformDistribution(low=1, high=1),
"e": IntLogUniformDistribution(low=1, high=1),
"f": CategoricalDistribution([1]),
"g": FloatDistribution(low=1.0, high=1.0),
"h": FloatDistribution(low=1.0, high=1.0, log=True),
"i": FloatDistribution(low=1.0, high=1.0, step=1.0),
"j": IntDistribution(low=1, high=1),
"k": IntDistribution(low=1, high=1, log=True),
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = sampler_class()
study = optuna.study.create_study(sampler=sampler)
# We need to test the construction of the model, so we should set `n_trials >= 2`.
for _ in range(2):
trial = study.ask(fixed_distributions=relative_search_space)
study.tell(trial, 1.0)
for param_name in relative_search_space.keys():
assert trial.params[param_name] == 1
def test_sample_relative() -> None:
relative_search_space: Dict[str, BaseDistribution] = {
"a": UniformDistribution(low=0, high=5),
"b": CategoricalDistribution(choices=("foo", "bar", "baz")),
"c": IntUniformDistribution(low=20, high=50), # Not exist in `relative_params`.
}
relative_params = {
"a": 3.2,
"b": "baz",
}
unknown_param_value = 30
sampler = FixedSampler( # type: ignore
relative_search_space, relative_params, unknown_param_value
)
study = optuna.study.create_study(sampler=sampler)
def objective(trial: Trial) -> float:
# Predefined parameters are sampled by `sample_relative()` method.
assert trial.suggest_uniform("a", 0, 5) == 3.2
assert trial.suggest_categorical("b", ["foo", "bar", "baz"]) == "baz"
# Other parameters are sampled by `sample_independent()` method.
assert trial.suggest_int("c", 20, 50) == unknown_param_value
assert trial.suggest_loguniform("d", 1, 100) == unknown_param_value
assert trial.suggest_uniform("e", 20, 40) == unknown_param_value
return 0.0
study.optimize(objective, n_trials=10, catch=())
for trial in study.trials:
assert trial.params == {"a": 3.2, "b": "baz", "c": 30, "d": 30, "e": 30}
def test_set_trial_param_to_check_distribution_json():
# type: () -> None
example_distributions = {
"x": UniformDistribution(low=1.0, high=2.0),
"y": CategoricalDistribution(choices=("Otemachi", "Tokyo", "Ginza")),
} # type: Dict[str, BaseDistribution]
storage = create_test_storage()
session = storage.scoped_session()
study_id = storage.create_new_study()
trial_id = storage.create_new_trial(study_id)
storage.set_trial_param(trial_id, "x", 1.5, example_distributions["x"])
storage.set_trial_param(trial_id, "y", 2, example_distributions["y"])
# test setting new name
result_1 = session.query(TrialParamModel).filter(
TrialParamModel.param_name == "x").one()
assert json_to_distribution(
result_1.distribution_json) == example_distributions["x"]
result_2 = session.query(TrialParamModel).filter(
TrialParamModel.param_name == "y").one()
assert json_to_distribution(
result_2.distribution_json) == example_distributions["y"]
def test_distributions(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
def objective(trial):
# type: (Trial) -> float
trial.suggest_uniform("a", 0, 10)
trial.suggest_loguniform("b", 0.1, 10)
trial.suggest_discrete_uniform("c", 0, 10, 1)
trial.suggest_int("d", 0, 10)
trial.suggest_categorical("e", ["foo", "bar", "baz"])
trial.suggest_int("f", 1, 10, log=True)
return 1.0
study = create_study(storage_init_func())
study.optimize(objective, n_trials=1)
assert study.best_trial.distributions == {
"a": UniformDistribution(low=0, high=10),
"b": LogUniformDistribution(low=0.1, high=10),
"c": DiscreteUniformDistribution(low=0, high=10, q=1),
"d": IntUniformDistribution(low=0, high=10),
"e": CategoricalDistribution(choices=("foo", "bar", "baz")),
"f": IntLogUniformDistribution(low=1, high=10),
}
def test_set_trial_param_to_check_distribution_json():
# type: () -> None
example_distributions = {
'x': UniformDistribution(low=1., high=2.),
'y': CategoricalDistribution(choices=('Otemachi', 'Tokyo', 'Ginza'))
} # type: Dict[str, BaseDistribution]
storage = create_test_storage()
session = storage.scoped_session()
study_id = storage.create_new_study_id()
trial_id = storage.create_new_trial_id(study_id)
storage.set_trial_param(trial_id, 'x', 1.5, example_distributions['x'])
storage.set_trial_param(trial_id, 'y', 2, example_distributions['y'])
# test setting new name
result_1 = session.query(TrialParamModel). \
filter(TrialParamModel.param_name == 'x').one()
assert json_to_distribution(
result_1.distribution_json) == example_distributions['x']
result_2 = session.query(TrialParamModel). \
filter(TrialParamModel.param_name == 'y').one()
assert json_to_distribution(
result_2.distribution_json) == example_distributions['y']
def test_shap_importance_evaluator_with_infinite(inf_value: float) -> None:
# The test ensures that trials with infinite values are ignored to calculate importance scores.
n_trial = 10
seed = 13
# Importance scores are calculated without a trial with an inf value.
study = create_study(sampler=RandomSampler(seed=seed))
study.optimize(objective, n_trials=n_trial)
evaluator = ShapleyImportanceEvaluator(seed=seed)
param_importance_without_inf = evaluator.evaluate(study)
# A trial with an inf value is added into the study manually.
study.add_trial(
create_trial(
value=inf_value,
params={"x1": 1.0, "x2": 1.0, "x3": 3.0, "x4": 0.1},
distributions={
"x1": FloatDistribution(low=0.1, high=3),
"x2": FloatDistribution(low=0.1, high=3, log=True),
"x3": IntDistribution(low=2, high=4, log=True),
"x4": CategoricalDistribution([0.1, 1, 10]),
},
)
)
# Importance scores are calculated with a trial with an inf value.
param_importance_with_inf = evaluator.evaluate(study)
# Obtained importance scores should be the same between with inf and without inf,
# because the last trial whose objective value is an inf is ignored.
assert param_importance_with_inf == param_importance_without_inf
def create_optuna_distribution_from_config(
config: MutableMapping[str, Any]) -> BaseDistribution:
kwargs = dict(config)
if isinstance(config["type"], str):
kwargs["type"] = DistributionType[config["type"]]
param = DistributionConfig(**kwargs)
if param.type == DistributionType.categorical:
assert param.choices is not None
return CategoricalDistribution(param.choices)
if param.type == DistributionType.int:
assert param.low is not None
assert param.high is not None
if param.log:
return IntLogUniformDistribution(int(param.low), int(param.high))
step = int(param.step) if param.step is not None else 1
return IntUniformDistribution(int(param.low),
int(param.high),
step=step)
if param.type == DistributionType.float:
assert param.low is not None
assert param.high is not None
if param.log:
return LogUniformDistribution(param.low, param.high)
if param.step is not None:
return DiscreteUniformDistribution(param.low, param.high,
param.step)
return UniformDistribution(param.low, param.high)
raise NotImplementedError(
f"{param.type} is not supported by Optuna sweeper.")
def test_sample_relative():
# type: () -> None
relative_search_space = {
'a': UniformDistribution(low=0, high=5),
'b': CategoricalDistribution(choices=('foo', 'bar', 'baz')),
'c': IntUniformDistribution(low=20, high=50), # Not exist in `relative_params`.
}
relative_params = {
'a': 3.2,
'b': 'baz',
}
unknown_param_value = 30
sampler = FixedSampler( # type: ignore
relative_search_space, relative_params, unknown_param_value)
study = optuna.study.create_study(sampler=sampler)
def objective(trial):
# type: (Trial) -> float
# Predefined parameters are sampled by `sample_relative()` method.
assert trial.suggest_uniform('a', 0, 5) == 3.2
assert trial.suggest_categorical('b', ['foo', 'bar', 'baz']) == 'baz'
# Other parameters are sampled by `sample_independent()` method.
assert trial.suggest_int('c', 20, 50) == unknown_param_value
assert trial.suggest_loguniform('d', 1, 100) == unknown_param_value
assert trial.suggest_uniform('e', 20, 40) == unknown_param_value
return 0.0
study.optimize(objective, n_trials=10, catch=())
for trial in study.trials:
assert trial.params == {'a': 3.2, 'b': 'baz', 'c': 30, 'd': 30, 'e': 30}
def create_optuna_distribution_from_override(override: Override) -> Any:
value = override.value()
if not override.is_sweep_override():
return value
choices: List[CategoricalChoiceType] = []
if override.is_choice_sweep():
assert isinstance(value, ChoiceSweep)
for x in override.sweep_iterator(transformer=Transformer.encode):
assert isinstance(
x, (str, int, float, bool)
), f"A choice sweep expects str, int, float, or bool type. Got {type(x)}."
choices.append(x)
return CategoricalDistribution(choices)
if override.is_range_sweep():
assert isinstance(value, RangeSweep)
assert value.start is not None
assert value.stop is not None
if value.shuffle:
for x in override.sweep_iterator(transformer=Transformer.encode):
assert isinstance(
x, (str, int, float, bool)
), f"A choice sweep expects str, int, float, or bool type. Got {type(x)}."
choices.append(x)
return CategoricalDistribution(choices)
return IntUniformDistribution(int(value.start),
int(value.stop),
step=int(value.step))
if override.is_interval_sweep():
assert isinstance(value, IntervalSweep)
assert value.start is not None
assert value.end is not None
if "log" in value.tags:
if isinstance(value.start, int) and isinstance(value.end, int):
return IntLogUniformDistribution(int(value.start),
int(value.end))
return LogUniformDistribution(value.start, value.end)
else:
if isinstance(value.start, int) and isinstance(value.end, int):
return IntUniformDistribution(value.start, value.end)
return UniformDistribution(value.start, value.end)
raise NotImplementedError(
f"{override} is not supported by Optuna sweeper.")