def test_sample_relative_n_startup_trials() -> None:
independent_sampler = DeterministicRelativeSampler({}, {})
sampler = optuna.samplers.CmaEsSampler(
n_startup_trials=2, independent_sampler=independent_sampler)
study = optuna.create_study(sampler=sampler)
def objective(t: optuna.Trial) -> float:
value = t.suggest_int("x", -1, 1) + t.suggest_int("y", -1, 1)
if t.number == 0:
raise Exception("first trial is failed")
return float(value)
# The independent sampler is used for Trial#0 (FAILED), Trial#1 (COMPLETE)
# and Trial#2 (COMPLETE). The CMA-ES is used for Trial#3 (COMPLETE).
with patch.object(independent_sampler,
"sample_independent",
wraps=independent_sampler.sample_independent
) as mock_independent, patch.object(
sampler,
"sample_relative",
wraps=sampler.sample_relative) as mock_relative:
study.optimize(objective, n_trials=4, catch=(Exception, ))
assert mock_independent.call_count == 6 # The objective function has two parameters.
assert mock_relative.call_count == 4
def test_init_cmaes_opts(use_separable_cma: bool, cma_class_str: str) -> None:
sampler = optuna.samplers.CmaEsSampler(
x0={
"x": 0,
"y": 0
},
sigma0=0.1,
seed=1,
n_startup_trials=1,
independent_sampler=DeterministicRelativeSampler({}, {}),
use_separable_cma=use_separable_cma,
)
study = optuna.create_study(sampler=sampler)
with patch(cma_class_str) as cma_class:
cma_obj = MagicMock()
cma_obj.ask.return_value = np.array((-1, -1))
cma_obj.generation = 0
cma_obj.population_size = 5
cma_class.return_value = cma_obj
study.optimize(lambda t: t.suggest_float("x", -1, 1) + t.suggest_float(
"y", -1, 1),
n_trials=2)
assert cma_class.call_count == 1
_, actual_kwargs = cma_class.call_args
assert np.array_equal(actual_kwargs["mean"], np.array([0, 0]))
assert actual_kwargs["sigma"] == 0.1
assert np.allclose(actual_kwargs["bounds"], np.array([(-1, 1),
(-1, 1)]))
assert actual_kwargs["seed"] == np.random.RandomState(1).randint(
1, 2**32)
assert actual_kwargs["n_max_resampling"] == 10 * 2
assert actual_kwargs["population_size"] is None
def test_relative_sampling(storage_mode: str,
comm: CommunicatorBase) -> None:
relative_search_space = {
"x": distributions.UniformDistribution(low=-10, high=10),
"y": distributions.LogUniformDistribution(low=20, high=30),
"z": distributions.CategoricalDistribution(choices=(-1.0, 1.0)),
}
relative_params = {"x": 1.0, "y": 25.0, "z": -1.0}
sampler = DeterministicRelativeSampler(
relative_search_space,
relative_params # type: ignore
)
with MultiNodeStorageSupplier(storage_mode, comm) as storage:
study = TestChainerMNStudy._create_shared_study(storage,
comm,
sampler=sampler)
mn_study = ChainerMNStudy(study, comm)
# Invoke optimize.
n_trials = 20
func = Func()
mn_study.optimize(func, n_trials=n_trials)
# Assert trial counts.
assert len(mn_study.trials) == n_trials
# Assert the parameters in `relative_params` have been suggested among all nodes.
for trial in mn_study.trials:
assert trial.params == relative_params
def test_init_cma_opts():
# type: () -> None
sampler = optuna.integration.CmaEsSampler(
x0={"x": 0, "y": 0},
sigma0=0.1,
cma_stds={"x": 1, "y": 1},
seed=1,
cma_opts={"popsize": 5},
independent_sampler=DeterministicRelativeSampler({}, {}),
)
study = optuna.create_study(sampler=sampler)
with patch("optuna.integration.cma._Optimizer") as mock_obj:
mock_obj.ask.return_value = {"x": -1, "y": -1}
study.optimize(
lambda t: t.suggest_int("x", -1, 1) + t.suggest_int("y", -1, 1), n_trials=2
)
assert mock_obj.mock_calls[0] == call(
{
"x": IntUniformDistribution(low=-1, high=1),
"y": IntUniformDistribution(low=-1, high=1),
},
{"x": 0, "y": 0},
0.1,
{"x": 1, "y": 1},
{"popsize": 5, "seed": 1, "verbose": -2},
)
def test_sample_relative_1d() -> None:
independent_sampler = DeterministicRelativeSampler({}, {})
sampler = optuna.samplers.CmaEsSampler(independent_sampler=independent_sampler)
study = optuna.create_study(sampler=sampler)
# If search space is one dimensional, the independent sampler is always used.
with patch.object(
independent_sampler, "sample_independent", wraps=independent_sampler.sample_independent
) as mock_object:
study.optimize(lambda t: t.suggest_int("x", -1, 1), n_trials=2)
assert mock_object.call_count == 2
def test_relative_parameters(storage_mode: str) -> None:
relative_search_space = {
"x": UniformDistribution(low=5, high=6),
"y": UniformDistribution(low=5, high=6),
}
relative_params = {"x": 5.5, "y": 5.5, "z": 5.5}
sampler = DeterministicRelativeSampler(relative_search_space,
relative_params) # type: ignore
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
def create_trial() -> Trial:
return Trial(study,
study._storage.create_new_trial(study._study_id))
# Suggested from `relative_params`.
trial0 = create_trial()
distribution0 = UniformDistribution(low=0, high=100)
assert trial0._suggest("x", distribution0) == 5.5
# Not suggested from `relative_params` (due to unknown parameter name).
trial1 = create_trial()
distribution1 = distribution0
assert trial1._suggest("w", distribution1) != 5.5
# Not suggested from `relative_params` (due to incompatible value range).
trial2 = create_trial()
distribution2 = UniformDistribution(low=0, high=5)
assert trial2._suggest("x", distribution2) != 5.5
# Error (due to incompatible distribution class).
trial3 = create_trial()
distribution3 = IntUniformDistribution(low=1, high=100)
with pytest.raises(ValueError):
trial3._suggest("y", distribution3)
# Error ('z' is included in `relative_params` but not in `relative_search_space`).
trial4 = create_trial()
distribution4 = UniformDistribution(low=0, high=10)
with pytest.raises(ValueError):
trial4._suggest("z", distribution4)
# Error (due to incompatible distribution class).
trial5 = create_trial()
distribution5 = IntLogUniformDistribution(low=1, high=100)
with pytest.raises(ValueError):
trial5._suggest("y", distribution5)
def test_relative_parameters(storage_init_func):
# type: (typing.Callable[[], storages.BaseStorage]) -> None
relative_search_space = {
'x': distributions.UniformDistribution(low=5, high=6),
'y': distributions.UniformDistribution(low=5, high=6)
}
relative_params = {
'x': 5.5,
'y': 5.5,
'z': 5.5
}
sampler = DeterministicRelativeSampler(relative_search_space, relative_params) # type: ignore
study = create_study(storage=storage_init_func(), sampler=sampler)
def create_trial():
# type: () -> Trial
return Trial(study, study.storage.create_new_trial_id(study.study_id))
# Suggested from `relative_params`.
trial0 = create_trial()
distribution0 = distributions.UniformDistribution(low=0, high=100)
assert trial0._suggest('x', distribution0) == 5.5
# Not suggested from `relative_params` (due to unknown parameter name).
trial1 = create_trial()
distribution1 = distribution0
assert trial1._suggest('w', distribution1) != 5.5
# Not suggested from `relative_params` (due to incompatible value range).
trial2 = create_trial()
distribution2 = distributions.UniformDistribution(low=0, high=5)
assert trial2._suggest('x', distribution2) != 5.5
# Error (due to incompatible distribution class).
trial3 = create_trial()
distribution3 = distributions.IntUniformDistribution(low=1, high=100)
with pytest.raises(ValueError):
trial3._suggest('y', distribution3)
# Error ('z' is included in `relative_params` but not in `relative_search_space`).
trial4 = create_trial()
distribution4 = distributions.UniformDistribution(low=0, high=10)
with pytest.raises(ValueError):
trial4._suggest('z', distribution4)
def test_sample_relative_n_startup_trials() -> None:
independent_sampler = DeterministicRelativeSampler({}, {})
sampler = optuna.integration.SkoptSampler(
n_startup_trials=2, independent_sampler=independent_sampler
)
study = optuna.create_study(sampler=sampler)
# The independent sampler is used for Trial#0 and Trial#1.
# SkoptSampler is used for Trial#2.
with patch.object(
independent_sampler, "sample_independent", wraps=independent_sampler.sample_independent
) as mock_independent, patch.object(
sampler, "sample_relative", wraps=sampler.sample_relative
) as mock_relative:
study.optimize(lambda t: t.suggest_int("x", -1, 1) + t.suggest_int("y", -1, 1), n_trials=3)
assert mock_independent.call_count == 4 # The objective function has two parameters.
assert mock_relative.call_count == 3
def test_population_size_is_multiplied_when_enable_ipop() -> None:
inc_popsize = 2
sampler = optuna.samplers.CmaEsSampler(
x0={
"x": 0,
"y": 0
},
sigma0=0.1,
seed=1,
n_startup_trials=1,
independent_sampler=DeterministicRelativeSampler({}, {}),
restart_strategy="ipop",
inc_popsize=inc_popsize,
)
study = optuna.create_study(sampler=sampler)
def objective(trial: optuna.Trial) -> float:
_ = trial.suggest_uniform("x", -1, 1)
_ = trial.suggest_uniform("y", -1, 1)
return 1.0
with patch("optuna.samplers._cmaes.CMA") as cma_class_mock, patch(
"optuna.samplers._cmaes.pickle") as pickle_mock:
pickle_mock.dump.return_value = b"serialized object"
should_stop_mock = MagicMock()
should_stop_mock.return_value = True
cma_obj = CMA(
mean=np.array([-1, -1], dtype=float),
sigma=1.3,
bounds=np.array([[-1, 1], [-1, 1]], dtype=float),
)
cma_obj.should_stop = should_stop_mock
cma_class_mock.return_value = cma_obj
popsize = cma_obj.population_size
study.optimize(objective, n_trials=2 + popsize)
assert cma_obj.should_stop.call_count == 1
_, actual_kwargs = cma_class_mock.call_args
assert actual_kwargs["population_size"] == inc_popsize * popsize
def test_init_cma_opts():
# type: () -> None
sampler = optuna.integration.CmaEsSampler(
x0={
'x': 0,
'y': 0
},
sigma0=0.1,
cma_stds={
'x': 1,
'y': 1
},
seed=1,
cma_opts={'popsize': 5},
independent_sampler=DeterministicRelativeSampler({}, {}))
study = optuna.create_study(sampler=sampler)
with patch('optuna.integration.cma._Optimizer') as mock_obj:
mock_obj.ask.return_value = {'x': -1, 'y': -1}
study.optimize(lambda t: t.suggest_int('x', -1, 1) + t.suggest_int(
'y', -1, 1),
n_trials=2)
assert mock_obj.mock_calls[0] == call(
{
'x': IntUniformDistribution(low=-1, high=1),
'y': IntUniformDistribution(low=-1, high=1)
}, {
'x': 0,
'y': 0
}, 0.1, {
'x': 1,
'y': 1
}, {
'popsize': 5,
'seed': 1,
'verbose': -2
})
def test_init_cmaes_opts(self):
# type: () -> None
sampler = cmaes.sampler.CMASampler(
x0={
"x": 0,
"y": 0
},
sigma0=0.1,
seed=1,
n_startup_trials=1,
independent_sampler=DeterministicRelativeSampler({}, {}),
)
study = optuna.create_study(sampler=sampler)
with patch("cmaes.sampler.CMA") as cma_class:
cma_obj = MagicMock()
cma_obj.ask.return_value = np.array((-1, -1))
cma_obj.generation = 0
cma_obj.population_size = 5
cma_class.return_value = cma_obj
study.optimize(
lambda t: t.suggest_uniform("x", -1, 1) + t.suggest_uniform(
"y", -1, 1),
n_trials=2,
)
cma_class.assert_called_once()
actual_kwargs = cma_class.mock_calls[0].kwargs
assert np.array_equal(actual_kwargs["mean"], np.array([0, 0]))
assert actual_kwargs["sigma"] == 0.1
assert np.array_equal(actual_kwargs["bounds"],
np.array([(-1, 1), (-1, 1)]))
assert actual_kwargs["seed"] == np.random.RandomState(1).randint(
1, 2**32)
assert actual_kwargs["n_max_resampling"] == 10 * 2
