def test_empty_range_contains() -> None:
i = distributions.IntDistribution(low=1, high=1)
assert not i._contains(0)
assert i._contains(1)
assert not i._contains(2)
iq = distributions.IntDistribution(low=1, high=1, step=2)
assert not iq._contains(0)
assert iq._contains(1)
assert not iq._contains(2)
il = distributions.IntDistribution(low=1, high=1, log=True)
assert not il._contains(0)
assert il._contains(1)
assert not il._contains(2)
f = distributions.FloatDistribution(low=1.0, high=1.0)
assert not f._contains(0.9)
assert f._contains(1.0)
assert not f._contains(1.1)
fd = distributions.FloatDistribution(low=1.0, high=1.0, step=2.0)
assert not fd._contains(0.9)
assert fd._contains(1.0)
assert not fd._contains(1.1)
fl = distributions.FloatDistribution(low=1.0, high=1.0, log=True)
assert not fl._contains(0.9)
assert fl._contains(1.0)
assert not fl._contains(1.1)
def test_ask_distribution_conversion_noop() -> None:
fixed_distributions = {
"ud": distributions.FloatDistribution(low=0,
high=10,
log=False,
step=None),
"dud": distributions.FloatDistribution(low=0,
high=10,
log=False,
step=2),
"lud": distributions.FloatDistribution(low=1,
high=10,
log=True,
step=None),
"id": distributions.IntDistribution(low=0, high=10, log=False, step=1),
"idd": distributions.IntDistribution(low=0, high=10, log=False,
step=2),
"ild": distributions.IntDistribution(low=1, high=10, log=True, step=1),
"cd": distributions.CategoricalDistribution(choices=["a", "b", "c"]),
}
study = create_study()
trial = study.ask(fixed_distributions=fixed_distributions)
# Check fixed_distributions doesn't change.
assert trial.distributions == fixed_distributions
def test_convert_old_distribution_to_new_distribution_noop() -> None:
# No conversion happens for CategoricalDistribution.
cd = distributions.CategoricalDistribution(choices=["a", "b", "c"])
assert distributions._convert_old_distribution_to_new_distribution(
cd) == cd
# No conversion happens for new distributions.
fd = distributions.FloatDistribution(low=0, high=10, log=False, step=None)
assert distributions._convert_old_distribution_to_new_distribution(
fd) == fd
dfd = distributions.FloatDistribution(low=0, high=10, log=False, step=2)
assert distributions._convert_old_distribution_to_new_distribution(
dfd) == dfd
lfd = distributions.FloatDistribution(low=1, high=10, log=True, step=None)
assert distributions._convert_old_distribution_to_new_distribution(
lfd) == lfd
id = distributions.IntDistribution(low=0, high=10)
assert distributions._convert_old_distribution_to_new_distribution(
id) == id
idd = distributions.IntDistribution(low=0, high=10, step=2)
assert distributions._convert_old_distribution_to_new_distribution(
idd) == idd
ild = distributions.IntDistribution(low=1, high=10, log=True)
assert distributions._convert_old_distribution_to_new_distribution(
ild) == ild
def test_optuna_search_convert_deprecated_distribution() -> None:
param_dist = {
"ud": distributions.UniformDistribution(low=0, high=10),
"dud": distributions.DiscreteUniformDistribution(low=0, high=10, q=2),
"lud": distributions.LogUniformDistribution(low=1, high=10),
"id": distributions.IntUniformDistribution(low=0, high=10),
"idd": distributions.IntUniformDistribution(low=0, high=10, step=2),
"ild": distributions.IntLogUniformDistribution(low=1, high=10),
}
expected_param_dist = {
"ud": distributions.FloatDistribution(low=0, high=10, log=False, step=None),
"dud": distributions.FloatDistribution(low=0, high=10, log=False, step=2),
"lud": distributions.FloatDistribution(low=1, high=10, log=True, step=None),
"id": distributions.IntDistribution(low=0, high=10, log=False, step=1),
"idd": distributions.IntDistribution(low=0, high=10, log=False, step=2),
"ild": distributions.IntDistribution(low=1, high=10, log=True, step=1),
}
optuna_search = integration.OptunaSearchCV(
KernelDensity(),
param_dist,
)
assert optuna_search.param_distributions == expected_param_dist
# It confirms that ask doesn't convert non-deprecated distributions.
optuna_search = integration.OptunaSearchCV(
KernelDensity(),
expected_param_dist,
)
assert optuna_search.param_distributions == expected_param_dist
def test_eq_ne_hash() -> None:
# Two instances of a class are regarded as equivalent if the fields have the same values.
for d in EXAMPLE_DISTRIBUTIONS.values():
d_copy = copy.deepcopy(d)
assert d == d_copy
assert hash(d) == hash(d_copy)
# Different field values.
di0 = distributions.FloatDistribution(low=1, high=2)
di1 = distributions.FloatDistribution(low=1, high=3)
assert di0 != di1
# Different distribution classes.
di2 = distributions.IntDistribution(low=1, high=2)
assert di0 != di2
# Different field values.
d0 = distributions.UniformDistribution(low=1, high=2)
d1 = distributions.UniformDistribution(low=1, high=3)
assert d0 != d1
# Different distribution classes.
d2 = distributions.IntUniformDistribution(low=1, high=2)
assert d0 != d2
def test_relative_sampling(storage_mode: str,
comm: CommunicatorBase) -> None:
relative_search_space = {
"x": distributions.FloatDistribution(low=-10, high=10),
"y": distributions.FloatDistribution(low=20, high=30, log=True),
"z": distributions.CategoricalDistribution(choices=(-1.0, 1.0)),
}
relative_params = {"x": 1.0, "y": 25.0, "z": -1.0}
sampler = DeterministicRelativeSampler(
relative_search_space,
relative_params # type: ignore
)
with MultiNodeStorageSupplier(storage_mode, comm) as storage:
study = TestChainerMNStudy._create_shared_study(storage,
comm,
sampler=sampler)
mn_study = ChainerMNStudy(study, comm)
# Invoke optimize.
n_trials = 20
func = Func()
mn_study.optimize(func, n_trials=n_trials)
# Assert trial counts.
assert len(mn_study.trials) == n_trials
# Assert the parameters in `relative_params` have been suggested among all nodes.
for trial in mn_study.trials:
assert trial.params == relative_params
def test_group() -> None:
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(multivariate=True, group=True)
study = optuna.create_study(sampler=sampler)
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_int("x", 0, 10), n_trials=2)
assert mock.call_count == 1
assert study.trials[-1].distributions == {"x": distributions.IntDistribution(low=0, high=10)}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10) + t.suggest_float("z", -3, 3), n_trials=1
)
assert mock.call_count == 1
assert study.trials[-1].distributions == {
"y": distributions.IntDistribution(low=0, high=10),
"z": distributions.FloatDistribution(low=-3, high=3),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10)
+ t.suggest_float("z", -3, 3)
+ t.suggest_float("u", 1e-2, 1e2, log=True)
+ bool(t.suggest_categorical("v", ["A", "B", "C"])),
n_trials=1,
)
assert mock.call_count == 2
assert study.trials[-1].distributions == {
"u": distributions.FloatDistribution(low=1e-2, high=1e2, log=True),
"v": distributions.CategoricalDistribution(choices=["A", "B", "C"]),
"y": distributions.IntDistribution(low=0, high=10),
"z": distributions.FloatDistribution(low=-3, high=3),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_float("u", 1e-2, 1e2, log=True), n_trials=1)
assert mock.call_count == 3
assert study.trials[-1].distributions == {
"u": distributions.FloatDistribution(low=1e-2, high=1e2, log=True)
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10) + t.suggest_int("w", 2, 8, log=True), n_trials=1
)
assert mock.call_count == 4
assert study.trials[-1].distributions == {
"y": distributions.IntDistribution(low=0, high=10),
"w": distributions.IntDistribution(low=2, high=8, log=True),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_int("x", 0, 10), n_trials=1)
assert mock.call_count == 6
assert study.trials[-1].distributions == {"x": distributions.IntDistribution(low=0, high=10)}
def test_empty_range_contains() -> None:
i = distributions.IntDistribution(low=1, high=1)
assert not i._contains(0)
assert i._contains(1)
assert not i._contains(2)
f = distributions.FloatDistribution(low=1.0, high=1.0)
assert not f._contains(0.9)
assert f._contains(1.0)
assert not f._contains(1.1)
fd = distributions.FloatDistribution(low=1.0, high=1.0, step=2.0)
assert not fd._contains(0.9)
assert fd._contains(1.0)
assert not fd._contains(1.1)
u = distributions.UniformDistribution(low=1.0, high=1.0)
assert not u._contains(0.9)
assert u._contains(1.0)
assert not u._contains(1.1)
lu = distributions.LogUniformDistribution(low=1.0, high=1.0)
assert not lu._contains(0.9)
assert lu._contains(1.0)
assert not lu._contains(1.1)
du = distributions.DiscreteUniformDistribution(low=1.0, high=1.0, q=2.0)
assert not du._contains(0.9)
assert du._contains(1.0)
assert not du._contains(1.1)
iu = distributions.IntUniformDistribution(low=1, high=1)
assert not iu._contains(0)
assert iu._contains(1)
assert not iu._contains(2)
iuq = distributions.IntUniformDistribution(low=1, high=1, step=2)
assert not iuq._contains(0)
assert iuq._contains(1)
assert not iuq._contains(2)
ilu = distributions.IntLogUniformDistribution(low=1, high=1)
assert not ilu._contains(0)
assert ilu._contains(1)
assert not ilu._contains(2)
iluq = distributions.IntLogUniformDistribution(low=1, high=1, step=2)
assert not iluq._contains(0)
assert iluq._contains(1)
assert not iluq._contains(2)
def test_create_trial_distribution_conversion() -> None:
fixed_params = {
"ud": 0,
"dud": 2,
"lud": 1,
"id": 0,
"idd": 2,
"ild": 1,
}
fixed_distributions = {
"ud": distributions.UniformDistribution(low=0, high=10),
"dud": distributions.DiscreteUniformDistribution(low=0, high=10, q=2),
"lud": distributions.LogUniformDistribution(low=1, high=10),
"id": distributions.IntUniformDistribution(low=0, high=10),
"idd": distributions.IntUniformDistribution(low=0, high=10, step=2),
"ild": distributions.IntLogUniformDistribution(low=1, high=10),
}
with pytest.warns(
FutureWarning,
match="See https://github.com/optuna/optuna/issues/2941",
) as record:
trial = create_trial(params=fixed_params,
distributions=fixed_distributions,
value=1)
assert len(record) == 6
expected_distributions = {
"ud": distributions.FloatDistribution(low=0,
high=10,
log=False,
step=None),
"dud": distributions.FloatDistribution(low=0,
high=10,
log=False,
step=2),
"lud": distributions.FloatDistribution(low=1,
high=10,
log=True,
step=None),
"id": distributions.IntDistribution(low=0, high=10, log=False, step=1),
"idd": distributions.IntDistribution(low=0, high=10, log=False,
step=2),
"ild": distributions.IntDistribution(low=1, high=10, log=True, step=1),
}
assert trial.distributions == expected_distributions
def test_optuna_search(enable_pruning: bool, fit_params: str) -> None:
X, y = make_blobs(n_samples=10)
est = SGDClassifier(max_iter=5, tol=1e-03)
param_dist = {
"alpha": distributions.FloatDistribution(1e-04, 1e03, log=True)
}
optuna_search = integration.OptunaSearchCV(
est,
param_dist,
cv=3,
enable_pruning=enable_pruning,
error_score="raise",
max_iter=5,
random_state=0,
return_train_score=True,
)
with pytest.raises(NotFittedError):
optuna_search._check_is_fitted()
if fit_params == "coef_init" and not enable_pruning:
optuna_search.fit(X, y, coef_init=np.ones((3, 2), dtype=np.float64))
else:
optuna_search.fit(X, y)
optuna_search.trials_dataframe()
optuna_search.decision_function(X)
optuna_search.predict(X)
optuna_search.score(X, y)
def test_calculate_shape_check(
mus: np.ndarray,
prior_weight: float,
prior: bool,
magic_clip: bool,
endpoints: bool,
multivariate: bool,
) -> None:
parameters = _ParzenEstimatorParameters(
prior_weight=prior_weight,
consider_prior=prior,
consider_magic_clip=magic_clip,
consider_endpoints=endpoints,
weights=default_weights,
multivariate=multivariate,
)
mpe = _ParzenEstimator({"a": mus},
{"a": distributions.FloatDistribution(-1.0, 1.0)},
parameters)
s_weights, s_mus, s_sigmas = mpe._weights, mpe._mus["a"], mpe._sigmas["a"]
# Result contains an additional value for a prior distribution if prior is True or
# len(mus) == 0 (in this case, prior is always used).
assert s_mus is not None
assert s_sigmas is not None
assert len(
s_weights) == len(mus) + int(prior) if len(mus) > 0 else len(mus) + 1
assert len(
s_mus) == len(mus) + int(prior) if len(mus) > 0 else len(mus) + 1
assert len(
s_sigmas) == len(mus) + int(prior) if len(mus) > 0 else len(mus) + 1
def test_suggest_with_step_parzen_estimator(multivariate: bool) -> None:
parameters = _ParzenEstimatorParameters(
consider_prior=False,
prior_weight=0.0,
consider_magic_clip=False,
consider_endpoints=False,
weights=lambda x: np.arange(x) + 1.0,
multivariate=multivariate,
)
# Define search space for distribution with step argument and true ranges
search_space = {
"c": distributions.FloatDistribution(low=1.0, high=7.0, step=3.0),
"d": distributions.IntDistribution(low=1, high=5, step=2),
}
multivariate_samples = {"c": np.array([4]), "d": np.array([3])}
valid_ranges = {
"c": set(np.arange(1.0, 10.0, 3.0)),
"d": set(np.arange(1, 7, 2))
}
sigmas0 = 1 if multivariate else None
with patch(_PRECOMPUTE_SIGMAS0, return_value=sigmas0):
mpe = _ParzenEstimator(multivariate_samples, search_space, parameters)
# Draw 10 samples, and check if all valid values are sampled.
output_samples = mpe.sample(np.random.RandomState(0), 10)
for param_name in output_samples:
assert set(output_samples[param_name]) == valid_ranges[param_name]
def test_optuna_search_properties() -> None:
X, y = make_blobs(n_samples=10)
est = LogisticRegression(tol=1e-03)
param_dist = {"C": distributions.FloatDistribution(1e-04, 1e03, log=True)}
optuna_search = integration.OptunaSearchCV(est,
param_dist,
cv=3,
error_score="raise",
random_state=0,
return_train_score=True)
optuna_search.fit(X, y)
optuna_search.set_user_attr("dataset", "blobs")
assert optuna_search._estimator_type == "classifier"
assert type(optuna_search.best_index_) == int
assert type(optuna_search.best_params_) == dict
assert optuna_search.best_score_ is not None
assert optuna_search.best_trial_ is not None
assert np.allclose(optuna_search.classes_, np.array([0, 1, 2]))
assert optuna_search.n_trials_ == 10
assert optuna_search.user_attrs_ == {"dataset": "blobs"}
assert type(optuna_search.predict_log_proba(X)) == np.ndarray
assert type(optuna_search.predict_proba(X)) == np.ndarray
def test_float_single(expected: bool, low: float, high: float, log: bool,
step: Optional[float]) -> None:
distribution = distributions.FloatDistribution(low=low,
high=high,
log=log,
step=step)
assert distribution.single() == expected
def test_callbacks() -> None:
callbacks = []
for _ in range(2):
callback = MagicMock()
callback.__call__ = MagicMock(return_value=None) # type: ignore
callbacks.append(callback)
n_trials = 5
X, y = make_blobs(n_samples=10)
est = SGDClassifier(max_iter=5, tol=1e-03)
param_dist = {
"alpha": distributions.FloatDistribution(1e-04, 1e03, log=True)
}
optuna_search = integration.OptunaSearchCV(
est,
param_dist,
cv=3,
enable_pruning=True,
max_iter=5,
n_trials=n_trials,
error_score=np.nan,
callbacks=callbacks, # type: ignore
)
optuna_search.fit(X, y)
for callback in callbacks:
for trial in optuna_search.trials_:
callback.assert_any_call(optuna_search.study_, trial)
assert callback.call_count == n_trials
def test_float_contains(expected: bool, value: float,
step: Optional[float]) -> None:
with warnings.catch_warnings():
# When `step` is 2.0, UserWarning will be raised since the range is not divisible by 2.
# The range will be replaced with [2.0, 6.0].
warnings.simplefilter("ignore", category=UserWarning)
f = distributions.FloatDistribution(low=2.0, high=7.0, step=step)
assert f._contains(value) == expected
def test_float_init_error() -> None:
# Empty distributions cannot be instantiated.
with pytest.raises(ValueError):
distributions.FloatDistribution(low=0.0, high=-100.0)
with pytest.raises(ValueError):
distributions.FloatDistribution(low=7.3, high=7.2, log=True)
with pytest.raises(ValueError):
distributions.FloatDistribution(low=-30.0, high=-40.0, step=2.5)
# 'step' must be None when 'log' is True.
with pytest.raises(ValueError):
distributions.FloatDistribution(low=1.0,
high=100.0,
log=True,
step=0.5)
# 'step' should be positive.
with pytest.raises(ValueError):
distributions.FloatDistribution(low=1.0, high=10.0, step=0)
with pytest.raises(ValueError):
distributions.FloatDistribution(low=1.0, high=100.0, step=-1)
def test_ask_fixed_search_space() -> None:
fixed_distributions = {
"x": distributions.FloatDistribution(0, 1),
"y": distributions.CategoricalDistribution(["bacon", "spam"]),
}
study = create_study()
trial = study.ask(fixed_distributions=fixed_distributions)
params = trial.params
assert len(trial.params) == 2
assert 0 <= params["x"] < 1
assert params["y"] in ["bacon", "spam"]
def test_invalid_weights(weights: Callable[[int], np.ndarray]) -> None:
parameters = _ParzenEstimatorParameters(
prior_weight=1.0,
consider_prior=False,
consider_magic_clip=False,
consider_endpoints=False,
weights=weights,
multivariate=False,
)
with pytest.raises(ValueError):
_ParzenEstimator({"a": np.asarray([0.0])},
{"a": distributions.FloatDistribution(-1.0, 1.0)},
parameters)
def test_infer_relative_search_space() -> None:
sampler = TPESampler()
search_space = {
"a": distributions.FloatDistribution(1.0, 100.0),
"b": distributions.FloatDistribution(1.0, 100.0, log=True),
"c": distributions.FloatDistribution(1.0, 100.0, step=3.0),
"d": distributions.IntDistribution(1, 100),
"e": distributions.IntDistribution(0, 100, step=2),
"f": distributions.IntDistribution(1, 100, log=True),
"g": distributions.CategoricalDistribution(["x", "y", "z"]),
}
def obj(t: Trial) -> float:
t.suggest_float("a", 1.0, 100.0)
t.suggest_float("b", 1.0, 100.0, log=True)
t.suggest_float("c", 1.0, 100.0, step=3.0)
t.suggest_int("d", 1, 100)
t.suggest_int("e", 0, 100, step=2)
t.suggest_int("f", 1, 100, log=True)
t.suggest_categorical("g", ["x", "y", "z"])
return 0.0
# Study and frozen-trial are not supposed to be accessed.
study1 = Mock(spec=[])
frozen_trial = Mock(spec=[])
assert sampler.infer_relative_search_space(study1, frozen_trial) == {}
study2 = optuna.create_study(sampler=sampler)
study2.optimize(obj, n_trials=1)
assert sampler.infer_relative_search_space(study2, study2.best_trial) == {}
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(multivariate=True)
study3 = optuna.create_study(sampler=sampler)
study3.optimize(obj, n_trials=1)
assert sampler.infer_relative_search_space(
study3, study3.best_trial) == search_space
def test_create_trial_distribution_conversion_noop() -> None:
fixed_params = {
"ud": 0,
"dud": 2,
"lud": 1,
"id": 0,
"idd": 2,
"ild": 1,
"cd": "a",
}
fixed_distributions = {
"ud": distributions.FloatDistribution(low=0,
high=10,
log=False,
step=None),
"dud": distributions.FloatDistribution(low=0,
high=10,
log=False,
step=2),
"lud": distributions.FloatDistribution(low=1,
high=10,
log=True,
step=None),
"id": distributions.IntDistribution(low=0, high=10, log=False, step=1),
"idd": distributions.IntDistribution(low=0, high=10, log=False,
step=2),
"ild": distributions.IntDistribution(low=1, high=10, log=True, step=1),
"cd": distributions.CategoricalDistribution(choices=["a", "b", "c"]),
}
trial = create_trial(params=fixed_params,
distributions=fixed_distributions,
value=1)
# Check fixed_distributions doesn't change.
assert trial.distributions == fixed_distributions
def test_convert_old_distribution_to_new_distribution() -> None:
ud = distributions.UniformDistribution(low=0, high=10)
assert distributions._convert_old_distribution_to_new_distribution(
ud) == distributions.FloatDistribution(low=0,
high=10,
log=False,
step=None)
dud = distributions.DiscreteUniformDistribution(low=0, high=10, q=2)
assert distributions._convert_old_distribution_to_new_distribution(
dud) == distributions.FloatDistribution(low=0,
high=10,
log=False,
step=2)
lud = distributions.LogUniformDistribution(low=1, high=10)
assert distributions._convert_old_distribution_to_new_distribution(
lud) == distributions.FloatDistribution(low=1,
high=10,
log=True,
step=None)
id = distributions.IntUniformDistribution(low=0, high=10)
assert distributions._convert_old_distribution_to_new_distribution(
id) == distributions.IntDistribution(low=0, high=10, log=False, step=1)
idd = distributions.IntUniformDistribution(low=0, high=10, step=2)
assert distributions._convert_old_distribution_to_new_distribution(
idd) == distributions.IntDistribution(low=0,
high=10,
log=False,
step=2)
ild = distributions.IntLogUniformDistribution(low=1, high=10)
assert distributions._convert_old_distribution_to_new_distribution(
ild) == distributions.IntDistribution(low=1, high=10, log=True, step=1)
def test_is_compatible() -> None:
sampler = optuna.integration.SkoptSampler()
study = optuna.create_study(sampler=sampler)
study.optimize(lambda t: t.suggest_float("p0", 0, 10), n_trials=1)
search_space = optuna.samplers.intersection_search_space(study)
assert search_space == {
"p0": distributions.FloatDistribution(low=0, high=10)
}
optimizer = optuna.integration.skopt._Optimizer(search_space, {})
# Compatible.
trial = _create_frozen_trial(
{"p0": 5}, {"p0": distributions.FloatDistribution(low=0, high=10)})
assert optimizer._is_compatible(trial)
# Compatible.
trial = _create_frozen_trial(
{"p0": 5}, {"p0": distributions.FloatDistribution(low=0, high=100)})
assert optimizer._is_compatible(trial)
# Compatible.
trial = _create_frozen_trial(
{
"p0": 5,
"p1": 7
},
{
"p0": distributions.FloatDistribution(low=0, high=10),
"p1": distributions.FloatDistribution(low=0, high=10),
},
)
assert optimizer._is_compatible(trial)
# Incompatible ('p0' doesn't exist).
trial = _create_frozen_trial(
{"p1": 5}, {"p1": distributions.FloatDistribution(low=0, high=10)})
assert not optimizer._is_compatible(trial)
# Incompatible (the value of 'p0' is out of range).
trial = _create_frozen_trial(
{"p0": 20}, {"p0": distributions.FloatDistribution(low=0, high=100)})
assert not optimizer._is_compatible(trial)
# Error (different distribution class).
trial = _create_frozen_trial(
{"p0": 5}, {"p0": distributions.IntDistribution(low=0, high=10)})
with pytest.raises(ValueError):
optimizer._is_compatible(trial)
def test_invalid_prior_weight(prior_weight: float, mus: np.ndarray) -> None:
parameters = _ParzenEstimatorParameters(
prior_weight=prior_weight,
consider_prior=True,
consider_magic_clip=False,
consider_endpoints=False,
weights=default_weights,
multivariate=False,
)
mpe = _ParzenEstimator({"a": mus},
{"a": distributions.FloatDistribution(-1.0, 1.0)},
parameters)
weights = mpe._weights
assert len(weights) == len(mus) + 1
# TODO(HideakiImamura): After modifying the body to raise an error, modify the test as well.
if prior_weight is None:
assert all([np.isnan(w) for w in weights])
def test_calculate(mus: np.ndarray, flags: Dict[str, bool],
expected: Dict[str, List[float]]) -> None:
parameters = _ParzenEstimatorParameters(
prior_weight=1.0,
consider_prior=flags["prior"],
consider_magic_clip=flags["magic_clip"],
consider_endpoints=flags["endpoints"],
weights=default_weights,
multivariate=False,
)
mpe = _ParzenEstimator({"a": mus},
{"a": distributions.FloatDistribution(-1.0, 1.0)},
parameters)
s_weights, s_mus, s_sigmas = mpe._weights, mpe._mus["a"], mpe._sigmas["a"]
# Result contains an additional value for a prior distribution if consider_prior is True.
assert isinstance(s_weights, np.ndarray)
assert isinstance(s_mus, np.ndarray)
assert isinstance(s_sigmas, np.ndarray)
np.testing.assert_almost_equal(s_weights, expected["weights"])
np.testing.assert_almost_equal(s_mus, expected["mus"])
np.testing.assert_almost_equal(s_sigmas, expected["sigmas"])
from optuna import distributions
_choices = (None, True, False, 0, 1, 0.0, 1.0, float("nan"), float("inf"),
-float("inf"), "", "a")
_choices_json = '[null, true, false, 0, 1, 0.0, 1.0, NaN, Infinity, -Infinity, "", "a"]'
EXAMPLE_DISTRIBUTIONS: Dict[str, Any] = {
"i": distributions.IntDistribution(low=1, high=9, log=False),
# i2 and i3 are identical to i, and tested for cases when `log` and `step` are omitted in json.
"i2": distributions.IntDistribution(low=1, high=9, log=False),
"i3": distributions.IntDistribution(low=1, high=9, log=False),
"il": distributions.IntDistribution(low=2, high=12, log=True),
"il2": distributions.IntDistribution(low=2, high=12, log=True),
"id": distributions.IntDistribution(low=1, high=9, log=False, step=2),
"id2": distributions.IntDistribution(low=1, high=9, log=False, step=2),
"f": distributions.FloatDistribution(low=1.0, high=2.0, log=False),
"fl": distributions.FloatDistribution(low=0.001, high=100.0, log=True),
"fd": distributions.FloatDistribution(low=1.0,
high=9.0,
log=False,
step=2.0),
"c1": distributions.CategoricalDistribution(choices=_choices),
"c2": distributions.CategoricalDistribution(choices=("Roppongi", "Azabu")),
"c3": distributions.CategoricalDistribution(choices=["Roppongi", "Azabu"]),
}
EXAMPLE_JSONS = {
"i":
'{"name": "IntDistribution", "attributes": {"low": 1, "high": 9}}',
"i2":
'{"name": "IntDistribution", "attributes": {"low": 1, "high": 9, "log": false}}',
from typing import Callable
from typing import Dict
from typing import List
from unittest.mock import patch
import numpy as np
import pytest
from optuna import distributions
from optuna.samplers._tpe.parzen_estimator import _ParzenEstimator
from optuna.samplers._tpe.parzen_estimator import _ParzenEstimatorParameters
from optuna.samplers._tpe.sampler import default_weights
SEARCH_SPACE = {
"a":
distributions.FloatDistribution(1.0, 100.0),
"b":
distributions.FloatDistribution(1.0, 100.0, log=True),
"c":
distributions.FloatDistribution(1.0, 100.0, step=3.0),
"d":
distributions.IntDistribution(1, 100),
"e":
distributions.IntDistribution(1, 100, log=True),
"f":
distributions.CategoricalDistribution(["x", "y", "z"]),
"g":
distributions.CategoricalDistribution(
[0.0, float("inf"), float("nan"), None]),
}
def test_check_distribution_compatibility() -> None:
# test the same distribution
for key in EXAMPLE_JSONS:
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS[key], EXAMPLE_DISTRIBUTIONS[key])
# test different distribution classes
pytest.raises(
ValueError,
lambda: distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"], EXAMPLE_DISTRIBUTIONS["fl"]),
)
# test compatibility between IntDistributions.
distributions.check_distribution_compatibility(EXAMPLE_DISTRIBUTIONS["id"],
EXAMPLE_DISTRIBUTIONS["i"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"], EXAMPLE_DISTRIBUTIONS["il"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["il"], EXAMPLE_DISTRIBUTIONS["id"])
# test compatibility between FloatDistributions.
distributions.check_distribution_compatibility(EXAMPLE_DISTRIBUTIONS["fd"],
EXAMPLE_DISTRIBUTIONS["f"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["f"], EXAMPLE_DISTRIBUTIONS["fl"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fl"], EXAMPLE_DISTRIBUTIONS["fd"])
# test dynamic value range (CategoricalDistribution)
pytest.raises(
ValueError,
lambda: distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["c2"],
distributions.CategoricalDistribution(choices=("Roppongi",
"Akasaka")),
),
)
# test dynamic value range (except CategoricalDistribution)
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"],
distributions.IntDistribution(low=-3, high=2))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["il"],
distributions.IntDistribution(low=1, high=13, log=True))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["id"],
distributions.IntDistribution(low=-3, high=2, step=2))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["f"],
distributions.FloatDistribution(low=-3.0, high=-2.0))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fl"],
distributions.FloatDistribution(low=0.1, high=1.0, log=True))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fd"],
distributions.FloatDistribution(low=-1.0, high=11.0, step=0.5))
def test_check_distribution_compatibility() -> None:
# test the same distribution
for key in EXAMPLE_JSONS:
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS[key], EXAMPLE_DISTRIBUTIONS[key])
# We need to create new objects to compare NaNs.
# See https://github.com/optuna/optuna/pull/3567#pullrequestreview-974939837.
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS[key],
distributions.json_to_distribution(EXAMPLE_JSONS[key]))
# test different distribution classes
pytest.raises(
ValueError,
lambda: distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"], EXAMPLE_DISTRIBUTIONS["fl"]),
)
# test compatibility between IntDistributions.
distributions.check_distribution_compatibility(EXAMPLE_DISTRIBUTIONS["id"],
EXAMPLE_DISTRIBUTIONS["i"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"], EXAMPLE_DISTRIBUTIONS["il"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["il"], EXAMPLE_DISTRIBUTIONS["id"])
# test compatibility between FloatDistributions.
distributions.check_distribution_compatibility(EXAMPLE_DISTRIBUTIONS["fd"],
EXAMPLE_DISTRIBUTIONS["f"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["f"], EXAMPLE_DISTRIBUTIONS["fl"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fl"], EXAMPLE_DISTRIBUTIONS["fd"])
# test dynamic value range (CategoricalDistribution)
pytest.raises(
ValueError,
lambda: distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["c2"],
distributions.CategoricalDistribution(choices=("Roppongi",
"Akasaka")),
),
)
# test dynamic value range (except CategoricalDistribution)
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"],
distributions.IntDistribution(low=-3, high=2))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["il"],
distributions.IntDistribution(low=1, high=13, log=True))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["id"],
distributions.IntDistribution(low=-3, high=2, step=2))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["f"],
distributions.FloatDistribution(low=-3.0, high=-2.0))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fl"],
distributions.FloatDistribution(low=0.1, high=1.0, log=True))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fd"],
distributions.FloatDistribution(low=-1.0, high=11.0, step=0.5))
def test_float_internal_representation(value: float) -> None:
f = distributions.FloatDistribution(low=2.0, high=7.0)
assert f.to_external_repr(f.to_internal_repr(value)) == value
