def test_suggest_loguniform(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
with pytest.raises(ValueError):
LogUniformDistribution(low=1.0, high=0.9)
with pytest.raises(ValueError):
LogUniformDistribution(low=0.0, high=0.9)
mock = Mock()
mock.side_effect = [1.0, 2.0, 3.0]
sampler = samplers.RandomSampler()
with patch.object(sampler, "sample_independent", mock) as mock_object:
study = create_study(storage_init_func(), sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
distribution = LogUniformDistribution(low=0.1, high=4.0)
assert trial._suggest("x",
distribution) == 1.0 # Test suggesting a param.
assert trial._suggest(
"x", distribution) == 1.0 # Test suggesting the same param.
assert trial._suggest(
"y", distribution) == 3.0 # Test suggesting a different param.
assert trial.params == {"x": 1.0, "y": 3.0}
assert mock_object.call_count == 3
def test_search_space_transform_untransform_params() -> None:
search_space = {
"x0": DiscreteUniformDistribution(0, 1, q=0.2),
"x1": CategoricalDistribution(["foo", "bar", "baz", "qux"]),
"x2": IntLogUniformDistribution(1, 10),
"x3": CategoricalDistribution(["quux", "quuz"]),
"x4": UniformDistribution(2, 3),
"x5": LogUniformDistribution(1, 10),
"x6": IntUniformDistribution(2, 4),
"x7": CategoricalDistribution(["corge"]),
"x8": UniformDistribution(-2, -2),
"x9": LogUniformDistribution(1, 1),
}
params = {
"x0": 0.2,
"x1": "qux",
"x2": 1,
"x3": "quux",
"x4": 2.0,
"x5": 1.0,
"x6": 2,
"x7": "corge",
"x8": -2.0,
"x9": 1.0,
}
trans = _SearchSpaceTransform(search_space)
trans_params = trans.transform(params)
untrans_params = trans.untransform(trans_params)
for name in params.keys():
assert untrans_params[name] == params[name]
def test_suggest_loguniform(storage_mode: str) -> None:
with pytest.raises(ValueError):
LogUniformDistribution(low=1.0, high=0.9)
with pytest.raises(ValueError):
LogUniformDistribution(low=0.0, high=0.9)
mock = Mock()
mock.side_effect = [1.0, 2.0]
sampler = samplers.RandomSampler()
with patch.object(
sampler, "sample_independent",
mock) as mock_object, StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
distribution = LogUniformDistribution(low=0.1, high=4.0)
assert trial._suggest("x",
distribution) == 1.0 # Test suggesting a param.
assert trial._suggest(
"x", distribution) == 1.0 # Test suggesting the same param.
assert trial._suggest(
"y", distribution) == 2.0 # Test suggesting a different param.
assert trial.params == {"x": 1.0, "y": 2.0}
assert mock_object.call_count == 2
def test_plot_parallel_coordinate_unique_hyper_param() -> None:
# Test case when one unique value is suggested during the optimization.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"param_b": 30
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": LogUniformDistribution(1, 1000),
},
))
# Both hyperparameters contain unique values.
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 0.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, )
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
assert figure.data[0]["dimensions"][1]["values"] == (0.0, )
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (math.log10(30),
math.log10(30))
assert figure.data[0]["dimensions"][2]["values"] == (math.log10(30), )
assert figure.data[0]["dimensions"][2]["ticktext"] == ("30", )
assert figure.data[0]["dimensions"][2]["tickvals"] == (math.log10(30), )
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "preferred",
"param_b": 20
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": LogUniformDistribution(1, 1000),
},
))
# Still "category_a" contains unique suggested value during the optimization.
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
assert figure.data[0]["dimensions"][1]["values"] == (0.0, 0.0)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
def create_new_trial_object(trial_info_dict):
new_trial = optuna.trial.create_trial(
params = {"optimizer": trial_info_dict["optimizer"],
"weight_decay":trial_info_dict["weight_decay"],
"learning_rate":trial_info_dict["learning_rate"]},
distributions = {"optimizer" : CategoricalDistribution(choices=("Adam", "RMSprop", "SGD")),
"weight_decay": LogUniformDistribution(1e-5, 1e-1),
"learning_rate": LogUniformDistribution(1e-7, 1e-5)},
value= trial_info_dict["value"],)
new_trial.user_attrs = {"worker_id": trial_info_dict["worker_id"]}
return new_trial
def create_optuna_distribution_from_config(
config: MutableMapping[str, Any]) -> BaseDistribution:
kwargs = dict(config)
if isinstance(config["type"], str):
kwargs["type"] = DistributionType[config["type"]]
param = DistributionConfig(**kwargs)
if param.type == DistributionType.categorical:
assert param.choices is not None
return CategoricalDistribution(param.choices)
if param.type == DistributionType.int:
assert param.low is not None
assert param.high is not None
if param.log:
return IntLogUniformDistribution(int(param.low), int(param.high))
step = int(param.step) if param.step is not None else 1
return IntUniformDistribution(int(param.low),
int(param.high),
step=step)
if param.type == DistributionType.float:
assert param.low is not None
assert param.high is not None
if param.log:
return LogUniformDistribution(param.low, param.high)
if param.step is not None:
return DiscreteUniformDistribution(param.low, param.high,
param.step)
return UniformDistribution(param.low, param.high)
raise NotImplementedError(
f"{param.type} is not supported by Optuna sweeper.")
def test_plot_slice_log_scale() -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={"x_linear": 1.0, "y_log": 1e-3},
distributions={
"x_linear": UniformDistribution(0.0, 3.0),
"y_log": LogUniformDistribution(1e-5, 1.0),
},
)
)
# Plot a parameter.
# TODO(ytknzw): Add more specific assertion with the test case.
figure = plot_slice(study, params=["y_log"])
assert figure.has_data()
figure = plot_slice(study, params=["x_linear"])
assert figure.has_data()
# Plot multiple parameters.
# TODO(ytknzw): Add more specific assertion with the test case.
figure = plot_slice(study)
assert len(figure) == 2
assert figure[0].has_data()
assert figure[1].has_data()
def create_optuna_distribution_from_override(override: Override) -> Any:
value = override.value()
if not override.is_sweep_override():
return value
if override.is_choice_sweep():
assert isinstance(value, ChoiceSweep)
choices = [
x for x in override.sweep_iterator(transformer=Transformer.encode)
]
return CategoricalDistribution(choices)
if override.is_range_sweep():
choices = [
x for x in override.sweep_iterator(transformer=Transformer.encode)
]
return CategoricalDistribution(choices)
if override.is_interval_sweep():
assert isinstance(value, IntervalSweep)
if "log" in value.tags:
if "int" in value.tags:
return IntLogUniformDistribution(value.start, value.end)
return LogUniformDistribution(value.start, value.end)
else:
if "int" in value.tags:
return IntUniformDistribution(value.start, value.end)
return UniformDistribution(value.start, value.end)
raise NotImplementedError(
"{} is not supported by Optuna sweeper.".format(override))
def test_plot_slice_log_scale() -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={
"x_linear": 1.0,
"y_log": 1e-3
},
distributions={
"x_linear": UniformDistribution(0.0, 3.0),
"y_log": LogUniformDistribution(1e-5, 1.0),
},
))
# Plot a parameter.
figure = plot_slice(study, params=["y_log"])
assert figure.layout["xaxis_type"] == "log"
figure = plot_slice(study, params=["x_linear"])
assert figure.layout["xaxis_type"] is None
# Plot multiple parameters.
figure = plot_slice(study)
assert figure.layout["xaxis_type"] is None
assert figure.layout["xaxis2_type"] == "log"
def suggest_float(
self,
name: str,
low: float,
high: float,
*,
step: Optional[float] = None,
log: bool = False,
) -> float:
if step is not None:
if log:
raise ValueError(
"The parameter `step` is not supported when `log` is True."
)
else:
return self._suggest(
name,
DiscreteUniformDistribution(low=low, high=high, q=step))
else:
if log:
return self._suggest(
name, LogUniformDistribution(low=low, high=high))
else:
return self._suggest(name,
UniformDistribution(low=low, high=high))
def test_distributions(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
def objective(trial):
# type: (Trial) -> float
trial.suggest_uniform("a", 0, 10)
trial.suggest_loguniform("b", 0.1, 10)
trial.suggest_discrete_uniform("c", 0, 10, 1)
trial.suggest_int("d", 0, 10)
trial.suggest_categorical("e", ["foo", "bar", "baz"])
trial.suggest_int("f", 1, 10, log=True)
return 1.0
study = create_study(storage_init_func())
study.optimize(objective, n_trials=1)
assert study.best_trial.distributions == {
"a": UniformDistribution(low=0, high=10),
"b": LogUniformDistribution(low=0.1, high=10),
"c": DiscreteUniformDistribution(low=0, high=10, q=1),
"d": IntUniformDistribution(low=0, high=10),
"e": CategoricalDistribution(choices=("foo", "bar", "baz")),
"f": IntLogUniformDistribution(low=1, high=10),
}
def test_plot_slice_log_scale() -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={"x_linear": 1.0, "y_log": 1e-3},
distributions={
"x_linear": UniformDistribution(0.0, 3.0),
"y_log": LogUniformDistribution(1e-5, 1.0),
},
)
)
# Plot a parameter.
figure = plot_slice(study, params=["y_log"])
assert len(figure.get_lines()) == 0
assert figure.xaxis.label.get_text() == "y_log"
figure = plot_slice(study, params=["x_linear"])
assert len(figure.get_lines()) == 0
assert figure.xaxis.label.get_text() == "x_linear"
# Plot multiple parameters.
figure = plot_slice(study)
assert len(figure) == 2
assert len(figure[0].get_lines()) == 0
assert len(figure[1].get_lines()) == 0
assert figure[0].xaxis.label.get_text() == "x_linear"
assert figure[1].xaxis.label.get_text() == "y_log"
def test_sample_single_distribution(
sampler_class: Callable[[], BaseSampler]) -> None:
relative_search_space = {
"a": UniformDistribution(low=1.0, high=1.0),
"b": LogUniformDistribution(low=1.0, high=1.0),
"c": DiscreteUniformDistribution(low=1.0, high=1.0, q=1.0),
"d": IntUniformDistribution(low=1, high=1),
"e": IntLogUniformDistribution(low=1, high=1),
"f": CategoricalDistribution([1]),
"g": FloatDistribution(low=1.0, high=1.0),
"h": FloatDistribution(low=1.0, high=1.0, log=True),
"i": FloatDistribution(low=1.0, high=1.0, step=1.0),
"j": IntDistribution(low=1, high=1),
"k": IntDistribution(low=1, high=1, log=True),
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = sampler_class()
study = optuna.study.create_study(sampler=sampler)
# We need to test the construction of the model, so we should set `n_trials >= 2`.
for _ in range(2):
trial = study.ask(fixed_distributions=relative_search_space)
study.tell(trial, 1.0)
for param_name in relative_search_space.keys():
assert trial.params[param_name] == 1
def test_set_and_get_trial_param(storage_init_func):
# type: (Callable[[], BaseStorage]) -> None
storage = storage_init_func()
# Setup test across multiple studies and trials.
study_id = storage.create_new_study()
trial_id_1 = storage.create_new_trial(study_id)
trial_id_2 = storage.create_new_trial(study_id)
trial_id_3 = storage.create_new_trial(storage.create_new_study())
# Setup Distributions.
distribution_x = UniformDistribution(low=1.0, high=2.0)
distribution_y_1 = CategoricalDistribution(choices=('Shibuya', 'Ebisu',
'Meguro'))
distribution_y_2 = CategoricalDistribution(choices=('Shibuya', 'Shinsen'))
distribution_z = LogUniformDistribution(low=1.0, high=100.0)
# Test trial_1: setting new params.
assert storage.set_trial_param(trial_id_1, 'x', 0.5, distribution_x)
assert storage.set_trial_param(trial_id_1, 'y', 2, distribution_y_1)
# Test trial_1: getting params.
assert storage.get_trial_param(trial_id_1, 'x') == 0.5
assert storage.get_trial_param(trial_id_1, 'y') == 2
# Test trial_1: checking all params and external repr.
assert storage.get_trial(trial_id_1).params == {'x': 0.5, 'y': 'Meguro'}
# Test trial_1: setting existing name.
assert not storage.set_trial_param(trial_id_1, 'x', 0.6, distribution_x)
# Setup trial_2: setting new params (to the same study as trial_1).
assert storage.set_trial_param(trial_id_2, 'x', 0.3, distribution_x)
assert storage.set_trial_param(trial_id_2, 'z', 0.1, distribution_z)
# Test trial_2: getting params.
assert storage.get_trial_param(trial_id_2, 'x') == 0.3
assert storage.get_trial_param(trial_id_2, 'z') == 0.1
# Test trial_2: checking all params and external repr.
assert storage.get_trial(trial_id_2).params == {'x': 0.3, 'z': 0.1}
# Test trial_2: setting different distribution.
with pytest.raises(ValueError):
storage.set_trial_param(trial_id_2, 'x', 0.5, distribution_z)
# Test trial_2: setting CategoricalDistribution in different order.
with pytest.raises(ValueError):
storage.set_trial_param(
trial_id_2, 'y', 2,
CategoricalDistribution(choices=('Meguro', 'Shibuya', 'Ebisu')))
# Setup trial_3: setting new params (to different study from trial_1).
if isinstance(storage, InMemoryStorage):
with pytest.raises(ValueError):
# InMemoryStorage shares the same study if create_new_study is additionally invoked.
# Thus, the following line should fail due to distribution incompatibility.
storage.set_trial_param(trial_id_3, 'y', 1, distribution_y_2)
else:
assert storage.set_trial_param(trial_id_3, 'y', 1, distribution_y_2)
assert storage.get_trial_param(trial_id_3, 'y') == 1
assert storage.get_trial(trial_id_3).params == {'y': 'Shinsen'}
def test_set_and_get_trial_param(storage_init_func):
# type: (Callable[[], BaseStorage]) -> None
storage = storage_init_func()
# Setup test across multiple studies and trials.
study_id = storage.create_new_study()
trial_id_1 = storage.create_new_trial(study_id)
trial_id_2 = storage.create_new_trial(study_id)
trial_id_3 = storage.create_new_trial(storage.create_new_study())
# Setup Distributions.
distribution_x = UniformDistribution(low=1.0, high=2.0)
distribution_y_1 = CategoricalDistribution(choices=("Shibuya", "Ebisu",
"Meguro"))
distribution_y_2 = CategoricalDistribution(choices=("Shibuya", "Shinsen"))
distribution_z = LogUniformDistribution(low=1.0, high=100.0)
# Test trial_1: setting new params.
assert storage.set_trial_param(trial_id_1, "x", 0.5, distribution_x)
assert storage.set_trial_param(trial_id_1, "y", 2, distribution_y_1)
# Test trial_1: getting params.
assert storage.get_trial_param(trial_id_1, "x") == 0.5
assert storage.get_trial_param(trial_id_1, "y") == 2
# Test trial_1: checking all params and external repr.
assert storage.get_trial(trial_id_1).params == {"x": 0.5, "y": "Meguro"}
# Test trial_1: setting existing name.
assert not storage.set_trial_param(trial_id_1, "x", 0.6, distribution_x)
# Setup trial_2: setting new params (to the same study as trial_1).
assert storage.set_trial_param(trial_id_2, "x", 0.3, distribution_x)
assert storage.set_trial_param(trial_id_2, "z", 0.1, distribution_z)
# Test trial_2: getting params.
assert storage.get_trial_param(trial_id_2, "x") == 0.3
assert storage.get_trial_param(trial_id_2, "z") == 0.1
# Test trial_2: checking all params and external repr.
assert storage.get_trial(trial_id_2).params == {"x": 0.3, "z": 0.1}
# Test trial_2: setting different distribution.
with pytest.raises(ValueError):
storage.set_trial_param(trial_id_2, "x", 0.5, distribution_z)
# Test trial_2: setting CategoricalDistribution in different order.
with pytest.raises(ValueError):
storage.set_trial_param(
trial_id_2, "y", 2,
CategoricalDistribution(choices=("Meguro", "Shibuya", "Ebisu")))
# Setup trial_3: setting new params (to different study from trial_1).
if isinstance(storage, InMemoryStorage):
pass
else:
assert storage.set_trial_param(trial_id_3, "y", 1, distribution_y_2)
assert storage.get_trial_param(trial_id_3, "y") == 1
assert storage.get_trial(trial_id_3).params == {"y": "Shinsen"}
def search_space() -> Dict[str, BaseDistribution]:
return {
"c": CategoricalDistribution(("a", "b")),
"d": DiscreteUniformDistribution(-1, 9, 2),
"i": IntUniformDistribution(-1, 1),
"ii": IntUniformDistribution(-1, 3, 2),
"l": LogUniformDistribution(0.001, 0.1),
"u": UniformDistribution(-2, 2),
}
def search_space():
# type: () -> Dict[str, BaseDistribution]
return {
'c': CategoricalDistribution(('a', 'b')),
'd': DiscreteUniformDistribution(-1, 9, 2),
'i': IntUniformDistribution(-1, 1),
'l': LogUniformDistribution(0.001, 0.1),
'u': UniformDistribution(-2, 2),
}
def suggest_loguniform(self, name, low, high):
# type: (str, float, float) -> float
"""Suggest a value for the continuous parameter.
The value is sampled from the range :math:`[\\mathsf{low}, \\mathsf{high})`
in the log domain. When :math:`\\mathsf{low} = \\mathsf{high}`, the value of
:math:`\\mathsf{low}` will be returned.
Example:
Suggest penalty parameter ``C`` of `SVC <https://scikit-learn.org/stable/modules/
generated/sklearn.svm.SVC.html>`_.
.. testcode::
import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
import optuna
X, y = load_iris(return_X_y=True)
X_train, X_valid, y_train, y_valid = train_test_split(X, y)
def objective(trial):
c = trial.suggest_loguniform('c', 1e-5, 1e2)
clf = SVC(C=c, gamma='scale', random_state=0)
clf.fit(X_train, y_train)
return clf.score(X_valid, y_valid)
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=3)
Args:
name:
A parameter name.
low:
Lower endpoint of the range of suggested values. ``low`` is included in the range.
high:
Upper endpoint of the range of suggested values. ``high`` is excluded from the
range.
Returns:
A suggested float value.
"""
distribution = LogUniformDistribution(low=low, high=high)
self._check_distribution(name, distribution)
if distribution.low == distribution.high:
return self._set_new_param_or_get_existing(name, distribution.low, distribution)
return self._suggest(name, distribution)
def _generate_trial(generator: random.Random) -> FrozenTrial:
example_params = {
"paramA": (generator.uniform(0, 1), UniformDistribution(0, 1)),
"paramB": (generator.uniform(1, 2), LogUniformDistribution(1, 2)),
"paramC": (
generator.choice(["CatA", "CatB", "CatC"]),
CategoricalDistribution(("CatA", "CatB", "CatC")),
),
"paramD": (generator.uniform(-3, 0), UniformDistribution(-3, 0)),
"paramE":
(generator.choice([0.1, 0.2]), CategoricalDistribution((0.1, 0.2))),
}
example_attrs = {
"attrA": "valueA",
"attrB": 1,
"attrC": None,
"attrD": {
"baseline_score": 0.001,
"tags": ["image", "classification"]
},
}
state = generator.choice(ALL_STATES)
params = {}
distributions = {}
user_attrs = {}
system_attrs = {}
intermediate_values = {}
for key, (value, dist) in example_params.items():
if generator.choice([True, False]):
params[key] = value
distributions[key] = dist
for key, value in example_attrs.items():
if generator.choice([True, False]):
user_attrs["usr_" + key] = value
if generator.choice([True, False]):
system_attrs["sys_" + key] = value
for i in range(generator.randint(4, 10)):
if generator.choice([True, False]):
intermediate_values[i] = generator.uniform(-10, 10)
return FrozenTrial(
number=0, # dummy
state=state,
value=generator.uniform(-10, 10),
datetime_start=datetime.now(),
datetime_complete=datetime.now() if state.is_finished() else None,
params=params,
distributions=distributions,
user_attrs=user_attrs,
system_attrs=system_attrs,
intermediate_values=intermediate_values,
trial_id=0, # dummy
)
def test_plot_parallel_coordinate_log_params() -> None:
# Test with log params.
study_log_params = create_study()
study_log_params.add_trial(
create_trial(
value=0.0,
params={
"param_a": 1e-6,
"param_b": 10
},
distributions={
"param_a": LogUniformDistribution(1e-7, 1e-2),
"param_b": LogUniformDistribution(1, 1000),
},
))
study_log_params.add_trial(
create_trial(
value=1.0,
params={
"param_a": 2e-5,
"param_b": 200
},
distributions={
"param_a": LogUniformDistribution(1e-7, 1e-2),
"param_b": LogUniformDistribution(1, 1000),
},
))
study_log_params.add_trial(
create_trial(
value=0.1,
params={
"param_a": 1e-4,
"param_b": 30
},
distributions={
"param_a": LogUniformDistribution(1e-7, 1e-2),
"param_b": LogUniformDistribution(1, 1000),
},
))
figure = plot_parallel_coordinate(study_log_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0, 0.1)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (-6.0, -4.0)
assert figure.data[0]["dimensions"][1]["values"] == (-6, math.log10(2e-5),
-4)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("1e-06", "1e-05",
"0.0001")
assert figure.data[0]["dimensions"][1]["tickvals"] == (-6, -5, -4.0)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (1.0, math.log10(200))
assert figure.data[0]["dimensions"][2]["values"] == (1.0, math.log10(200),
math.log10(30))
assert figure.data[0]["dimensions"][2]["ticktext"] == ("10", "100", "200")
assert figure.data[0]["dimensions"][2]["tickvals"] == (1.0, 2.0,
math.log10(200))
def test_plot_parallel_coordinate_unique_hyper_param() -> None:
# Test case when one unique value is suggested during the optimization.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"param_b": 30
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": LogUniformDistribution(1, 1000),
},
))
# both hyper-parameter contains unique value
figure = plot_parallel_coordinate(study_categorical_params)
assert figure.has_data()
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "preferred",
"param_b": 20
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": LogUniformDistribution(1, 1000),
},
))
# still "category_a" contains unique suggested value during the optimization.
figure = plot_parallel_coordinate(study_categorical_params)
assert figure.has_data()
def test_frozen_trial_validate():
# type: () -> None
# Valid.
valid_trial = _create_frozen_trial()
valid_trial._validate()
# Invalid: `datetime_start` is not set.
invalid_trial = copy.copy(valid_trial)
invalid_trial.datetime_start = None
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: `state` is `RUNNING` and `datetime_complete` is set.
invalid_trial = copy.copy(valid_trial)
invalid_trial.state = TrialState.RUNNING
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: `state` is not `RUNNING` and `datetime_complete` is not set.
for state in [TrialState.COMPLETE, TrialState.PRUNED, TrialState.FAIL]:
invalid_trial = copy.copy(valid_trial)
invalid_trial.state = state
invalid_trial.datetime_complete = None
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: `state` is `COMPLETE` and `value` is not set.
invalid_trial = copy.copy(valid_trial)
invalid_trial.value = None
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: Inconsistent `params` and `distributions`
inconsistent_pairs = [
# `params` has an extra element.
({"x": 0.1, "y": 0.5}, {"x": UniformDistribution(0, 1)}),
# `distributions` has an extra element.
({"x": 0.1}, {"x": UniformDistribution(0, 1), "y": LogUniformDistribution(0, 1)}),
# The value of `x` isn't contained in the distribution.
({"x": -0.5}, {"x": UniformDistribution(0, 1)}),
] # type: List[Tuple[Dict[str, Any], Dict[str, BaseDistribution]]]
for params, distributions in inconsistent_pairs:
invalid_trial = copy.copy(valid_trial)
invalid_trial.params = params
invalid_trial.distributions = distributions
with pytest.raises(ValueError):
invalid_trial._validate()
def test_not_contained_param() -> None:
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": UniformDistribution(1.0, 10.0)},
)
with pytest.warns(UserWarning):
assert trial.suggest_float("x", 10.0, 100.0) == 1.0
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": LogUniformDistribution(1.0, 10.0)},
)
with pytest.warns(UserWarning):
assert trial.suggest_float("x", 10.0, 100.0, log=True) == 1.0
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": DiscreteUniformDistribution(1.0, 10.0, 1.0)},
)
with pytest.warns(UserWarning):
assert trial.suggest_float("x", 10.0, 100.0, step=1.0) == 1.0
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": IntUniformDistribution(1, 10)},
)
with pytest.warns(UserWarning):
assert trial.suggest_int("x", 10, 100) == 1
trial = create_trial(
value=0.2,
params={"x": 1},
distributions={"x": IntUniformDistribution(1, 10, 1)},
)
with pytest.warns(UserWarning):
assert trial.suggest_int("x", 10, 100, 1) == 1
trial = create_trial(
value=0.2,
params={"x": 1},
distributions={"x": IntLogUniformDistribution(1, 10)},
)
with pytest.warns(UserWarning):
assert trial.suggest_int("x", 10, 100, log=True) == 1
def create_optuna_distribution_from_override(override: Override) -> Any:
value = override.value()
if not override.is_sweep_override():
return value
choices: List[CategoricalChoiceType] = []
if override.is_choice_sweep():
assert isinstance(value, ChoiceSweep)
for x in override.sweep_iterator(transformer=Transformer.encode):
assert isinstance(
x, (str, int, float, bool)
), f"A choice sweep expects str, int, float, or bool type. Got {type(x)}."
choices.append(x)
return CategoricalDistribution(choices)
if override.is_range_sweep():
assert isinstance(value, RangeSweep)
assert value.start is not None
assert value.stop is not None
if value.shuffle:
for x in override.sweep_iterator(transformer=Transformer.encode):
assert isinstance(
x, (str, int, float, bool)
), f"A choice sweep expects str, int, float, or bool type. Got {type(x)}."
choices.append(x)
return CategoricalDistribution(choices)
return IntUniformDistribution(int(value.start),
int(value.stop),
step=int(value.step))
if override.is_interval_sweep():
assert isinstance(value, IntervalSweep)
assert value.start is not None
assert value.end is not None
if "log" in value.tags:
if isinstance(value.start, int) and isinstance(value.end, int):
return IntLogUniformDistribution(int(value.start),
int(value.end))
return LogUniformDistribution(value.start, value.end)
else:
if isinstance(value.start, int) and isinstance(value.end, int):
return IntUniformDistribution(value.start, value.end)
return UniformDistribution(value.start, value.end)
raise NotImplementedError(
f"{override} is not supported by Optuna sweeper.")
def restore_old_distribution(distribution_json: str) -> str:
distribution = json_to_distribution(distribution_json)
old_distribution: BaseDistribution
# Float distributions.
if isinstance(distribution, FloatDistribution):
if distribution.log:
old_distribution = LogUniformDistribution(
low=distribution.low,
high=distribution.high,
)
else:
if distribution.step is not None:
old_distribution = DiscreteUniformDistribution(
low=distribution.low,
high=distribution.high,
q=distribution.step,
)
else:
old_distribution = UniformDistribution(
low=distribution.low,
high=distribution.high,
)
# Integer distributions.
elif isinstance(distribution, IntDistribution):
if distribution.log:
old_distribution = IntLogUniformDistribution(
low=distribution.low,
high=distribution.high,
step=distribution.step,
)
else:
old_distribution = IntUniformDistribution(
low=distribution.low,
high=distribution.high,
step=distribution.step,
)
# Categorical distribution.
else:
old_distribution = distribution
return distribution_to_json(old_distribution)
def test_is_log_scale():
# type: () -> None
study = create_study()
study._append_trial(
value=0.0,
params={"param_linear": 1.0,},
distributions={"param_linear": UniformDistribution(0.0, 3.0),},
)
study._append_trial(
value=2.0,
params={"param_linear": 2.0, "param_log": 1e-3,},
distributions={
"param_linear": UniformDistribution(0.0, 3.0),
"param_log": LogUniformDistribution(1e-5, 1.0),
},
)
assert _is_log_scale(study.trials, "param_log")
assert not _is_log_scale(study.trials, "param_linear")
def test_frozen_trial_suggest_loguniform() -> None:
trial = FrozenTrial(
number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.datetime.now(),
datetime_complete=datetime.datetime.now(),
params={"x": 0.99},
distributions={"x": LogUniformDistribution(0.1, 1.0)},
user_attrs={},
system_attrs={},
intermediate_values={},
)
assert trial.suggest_loguniform("x", 0.1, 1.0) == 0.99
with pytest.raises(ValueError):
trial.suggest_loguniform("y", 0.0, 1.0)
def test_is_log_scale():
# type: () -> None
study = create_study()
study._append_trial(value=0.0,
params={
'param_linear': 1.0,
},
distributions={
'param_linear': UniformDistribution(0.0, 3.0),
})
study._append_trial(value=2.0,
params={
'param_linear': 2.0,
'param_log': 1e-3,
},
distributions={
'param_linear': UniformDistribution(0.0, 3.0),
'param_log': LogUniformDistribution(1e-5, 1.),
})
assert visualization._is_log_scale(study.trials, 'param_log')
assert not visualization._is_log_scale(study.trials, 'param_linear')
def test_is_log_scale() -> None:
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={"param_linear": 1.0},
distributions={"param_linear": UniformDistribution(0.0, 3.0)},
))
study.add_trial(
create_trial(
value=2.0,
params={
"param_linear": 2.0,
"param_log": 1e-3
},
distributions={
"param_linear": UniformDistribution(0.0, 3.0),
"param_log": LogUniformDistribution(1e-5, 1.0),
},
))
assert _is_log_scale(study.trials, "param_log")
assert not _is_log_scale(study.trials, "param_linear")
def test_distributions(storage_mode: str) -> None:
def objective(trial: Trial) -> float:
trial.suggest_float("a", 0, 10)
trial.suggest_float("b", 0.1, 10, log=True)
trial.suggest_float("c", 0, 10, step=1)
trial.suggest_int("d", 0, 10)
trial.suggest_categorical("e", ["foo", "bar", "baz"])
trial.suggest_int("f", 1, 10, log=True)
return 1.0
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage)
study.optimize(objective, n_trials=1)
assert study.best_trial.distributions == {
"a": UniformDistribution(low=0, high=10),
"b": LogUniformDistribution(low=0.1, high=10),
"c": DiscreteUniformDistribution(low=0, high=10, q=1),
"d": IntUniformDistribution(low=0, high=10),
"e": CategoricalDistribution(choices=("foo", "bar", "baz")),
"f": IntLogUniformDistribution(low=1, high=10),
}
