def test_sample_relative_n_startup_trials() -> None: independent_sampler = DeterministicRelativeSampler({}, {}) sampler = optuna.samplers.CmaEsSampler( n_startup_trials=2, independent_sampler=independent_sampler) study = optuna.create_study(sampler=sampler) def objective(t: optuna.Trial) -> float: value = t.suggest_int("x", -1, 1) + t.suggest_int("y", -1, 1) if t.number == 0: raise Exception("first trial is failed") return float(value) # The independent sampler is used for Trial#0 (FAILED), Trial#1 (COMPLETE) # and Trial#2 (COMPLETE). The CMA-ES is used for Trial#3 (COMPLETE). with patch.object(independent_sampler, "sample_independent", wraps=independent_sampler.sample_independent ) as mock_independent, patch.object( sampler, "sample_relative", wraps=sampler.sample_relative) as mock_relative: study.optimize(objective, n_trials=4, catch=(Exception, )) assert mock_independent.call_count == 6 # The objective function has two parameters. assert mock_relative.call_count == 4
def test_init_cmaes_opts(use_separable_cma: bool, cma_class_str: str) -> None: sampler = optuna.samplers.CmaEsSampler( x0={ "x": 0, "y": 0 }, sigma0=0.1, seed=1, n_startup_trials=1, independent_sampler=DeterministicRelativeSampler({}, {}), use_separable_cma=use_separable_cma, ) study = optuna.create_study(sampler=sampler) with patch(cma_class_str) as cma_class: cma_obj = MagicMock() cma_obj.ask.return_value = np.array((-1, -1)) cma_obj.generation = 0 cma_obj.population_size = 5 cma_class.return_value = cma_obj study.optimize(lambda t: t.suggest_float("x", -1, 1) + t.suggest_float( "y", -1, 1), n_trials=2) assert cma_class.call_count == 1 _, actual_kwargs = cma_class.call_args assert np.array_equal(actual_kwargs["mean"], np.array([0, 0])) assert actual_kwargs["sigma"] == 0.1 assert np.allclose(actual_kwargs["bounds"], np.array([(-1, 1), (-1, 1)])) assert actual_kwargs["seed"] == np.random.RandomState(1).randint( 1, 2**32) assert actual_kwargs["n_max_resampling"] == 10 * 2 assert actual_kwargs["population_size"] is None
def test_relative_sampling(storage_mode: str, comm: CommunicatorBase) -> None: relative_search_space = { "x": distributions.UniformDistribution(low=-10, high=10), "y": distributions.LogUniformDistribution(low=20, high=30), "z": distributions.CategoricalDistribution(choices=(-1.0, 1.0)), } relative_params = {"x": 1.0, "y": 25.0, "z": -1.0} sampler = DeterministicRelativeSampler( relative_search_space, relative_params # type: ignore ) with MultiNodeStorageSupplier(storage_mode, comm) as storage: study = TestChainerMNStudy._create_shared_study(storage, comm, sampler=sampler) mn_study = ChainerMNStudy(study, comm) # Invoke optimize. n_trials = 20 func = Func() mn_study.optimize(func, n_trials=n_trials) # Assert trial counts. assert len(mn_study.trials) == n_trials # Assert the parameters in `relative_params` have been suggested among all nodes. for trial in mn_study.trials: assert trial.params == relative_params
def test_init_cma_opts(): # type: () -> None sampler = optuna.integration.CmaEsSampler( x0={"x": 0, "y": 0}, sigma0=0.1, cma_stds={"x": 1, "y": 1}, seed=1, cma_opts={"popsize": 5}, independent_sampler=DeterministicRelativeSampler({}, {}), ) study = optuna.create_study(sampler=sampler) with patch("optuna.integration.cma._Optimizer") as mock_obj: mock_obj.ask.return_value = {"x": -1, "y": -1} study.optimize( lambda t: t.suggest_int("x", -1, 1) + t.suggest_int("y", -1, 1), n_trials=2 ) assert mock_obj.mock_calls[0] == call( { "x": IntUniformDistribution(low=-1, high=1), "y": IntUniformDistribution(low=-1, high=1), }, {"x": 0, "y": 0}, 0.1, {"x": 1, "y": 1}, {"popsize": 5, "seed": 1, "verbose": -2}, )
def test_sample_relative_1d() -> None: independent_sampler = DeterministicRelativeSampler({}, {}) sampler = optuna.samplers.CmaEsSampler(independent_sampler=independent_sampler) study = optuna.create_study(sampler=sampler) # If search space is one dimensional, the independent sampler is always used. with patch.object( independent_sampler, "sample_independent", wraps=independent_sampler.sample_independent ) as mock_object: study.optimize(lambda t: t.suggest_int("x", -1, 1), n_trials=2) assert mock_object.call_count == 2
def test_relative_parameters(storage_mode: str) -> None: relative_search_space = { "x": UniformDistribution(low=5, high=6), "y": UniformDistribution(low=5, high=6), } relative_params = {"x": 5.5, "y": 5.5, "z": 5.5} sampler = DeterministicRelativeSampler(relative_search_space, relative_params) # type: ignore with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) def create_trial() -> Trial: return Trial(study, study._storage.create_new_trial(study._study_id)) # Suggested from `relative_params`. trial0 = create_trial() distribution0 = UniformDistribution(low=0, high=100) assert trial0._suggest("x", distribution0) == 5.5 # Not suggested from `relative_params` (due to unknown parameter name). trial1 = create_trial() distribution1 = distribution0 assert trial1._suggest("w", distribution1) != 5.5 # Not suggested from `relative_params` (due to incompatible value range). trial2 = create_trial() distribution2 = UniformDistribution(low=0, high=5) assert trial2._suggest("x", distribution2) != 5.5 # Error (due to incompatible distribution class). trial3 = create_trial() distribution3 = IntUniformDistribution(low=1, high=100) with pytest.raises(ValueError): trial3._suggest("y", distribution3) # Error ('z' is included in `relative_params` but not in `relative_search_space`). trial4 = create_trial() distribution4 = UniformDistribution(low=0, high=10) with pytest.raises(ValueError): trial4._suggest("z", distribution4) # Error (due to incompatible distribution class). trial5 = create_trial() distribution5 = IntLogUniformDistribution(low=1, high=100) with pytest.raises(ValueError): trial5._suggest("y", distribution5)
def test_relative_parameters(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None relative_search_space = { 'x': distributions.UniformDistribution(low=5, high=6), 'y': distributions.UniformDistribution(low=5, high=6) } relative_params = { 'x': 5.5, 'y': 5.5, 'z': 5.5 } sampler = DeterministicRelativeSampler(relative_search_space, relative_params) # type: ignore study = create_study(storage=storage_init_func(), sampler=sampler) def create_trial(): # type: () -> Trial return Trial(study, study.storage.create_new_trial_id(study.study_id)) # Suggested from `relative_params`. trial0 = create_trial() distribution0 = distributions.UniformDistribution(low=0, high=100) assert trial0._suggest('x', distribution0) == 5.5 # Not suggested from `relative_params` (due to unknown parameter name). trial1 = create_trial() distribution1 = distribution0 assert trial1._suggest('w', distribution1) != 5.5 # Not suggested from `relative_params` (due to incompatible value range). trial2 = create_trial() distribution2 = distributions.UniformDistribution(low=0, high=5) assert trial2._suggest('x', distribution2) != 5.5 # Error (due to incompatible distribution class). trial3 = create_trial() distribution3 = distributions.IntUniformDistribution(low=1, high=100) with pytest.raises(ValueError): trial3._suggest('y', distribution3) # Error ('z' is included in `relative_params` but not in `relative_search_space`). trial4 = create_trial() distribution4 = distributions.UniformDistribution(low=0, high=10) with pytest.raises(ValueError): trial4._suggest('z', distribution4)
def test_sample_relative_n_startup_trials() -> None: independent_sampler = DeterministicRelativeSampler({}, {}) sampler = optuna.integration.SkoptSampler( n_startup_trials=2, independent_sampler=independent_sampler ) study = optuna.create_study(sampler=sampler) # The independent sampler is used for Trial#0 and Trial#1. # SkoptSampler is used for Trial#2. with patch.object( independent_sampler, "sample_independent", wraps=independent_sampler.sample_independent ) as mock_independent, patch.object( sampler, "sample_relative", wraps=sampler.sample_relative ) as mock_relative: study.optimize(lambda t: t.suggest_int("x", -1, 1) + t.suggest_int("y", -1, 1), n_trials=3) assert mock_independent.call_count == 4 # The objective function has two parameters. assert mock_relative.call_count == 3
def test_population_size_is_multiplied_when_enable_ipop() -> None: inc_popsize = 2 sampler = optuna.samplers.CmaEsSampler( x0={ "x": 0, "y": 0 }, sigma0=0.1, seed=1, n_startup_trials=1, independent_sampler=DeterministicRelativeSampler({}, {}), restart_strategy="ipop", inc_popsize=inc_popsize, ) study = optuna.create_study(sampler=sampler) def objective(trial: optuna.Trial) -> float: _ = trial.suggest_uniform("x", -1, 1) _ = trial.suggest_uniform("y", -1, 1) return 1.0 with patch("optuna.samplers._cmaes.CMA") as cma_class_mock, patch( "optuna.samplers._cmaes.pickle") as pickle_mock: pickle_mock.dump.return_value = b"serialized object" should_stop_mock = MagicMock() should_stop_mock.return_value = True cma_obj = CMA( mean=np.array([-1, -1], dtype=float), sigma=1.3, bounds=np.array([[-1, 1], [-1, 1]], dtype=float), ) cma_obj.should_stop = should_stop_mock cma_class_mock.return_value = cma_obj popsize = cma_obj.population_size study.optimize(objective, n_trials=2 + popsize) assert cma_obj.should_stop.call_count == 1 _, actual_kwargs = cma_class_mock.call_args assert actual_kwargs["population_size"] == inc_popsize * popsize
def test_init_cma_opts(): # type: () -> None sampler = optuna.integration.CmaEsSampler( x0={ 'x': 0, 'y': 0 }, sigma0=0.1, cma_stds={ 'x': 1, 'y': 1 }, seed=1, cma_opts={'popsize': 5}, independent_sampler=DeterministicRelativeSampler({}, {})) study = optuna.create_study(sampler=sampler) with patch('optuna.integration.cma._Optimizer') as mock_obj: mock_obj.ask.return_value = {'x': -1, 'y': -1} study.optimize(lambda t: t.suggest_int('x', -1, 1) + t.suggest_int( 'y', -1, 1), n_trials=2) assert mock_obj.mock_calls[0] == call( { 'x': IntUniformDistribution(low=-1, high=1), 'y': IntUniformDistribution(low=-1, high=1) }, { 'x': 0, 'y': 0 }, 0.1, { 'x': 1, 'y': 1 }, { 'popsize': 5, 'seed': 1, 'verbose': -2 })
def test_init_cmaes_opts(self): # type: () -> None sampler = cmaes.sampler.CMASampler( x0={ "x": 0, "y": 0 }, sigma0=0.1, seed=1, n_startup_trials=1, independent_sampler=DeterministicRelativeSampler({}, {}), ) study = optuna.create_study(sampler=sampler) with patch("cmaes.sampler.CMA") as cma_class: cma_obj = MagicMock() cma_obj.ask.return_value = np.array((-1, -1)) cma_obj.generation = 0 cma_obj.population_size = 5 cma_class.return_value = cma_obj study.optimize( lambda t: t.suggest_uniform("x", -1, 1) + t.suggest_uniform( "y", -1, 1), n_trials=2, ) cma_class.assert_called_once() actual_kwargs = cma_class.mock_calls[0].kwargs assert np.array_equal(actual_kwargs["mean"], np.array([0, 0])) assert actual_kwargs["sigma"] == 0.1 assert np.array_equal(actual_kwargs["bounds"], np.array([(-1, 1), (-1, 1)])) assert actual_kwargs["seed"] == np.random.RandomState(1).randint( 1, 2**32) assert actual_kwargs["n_max_resampling"] == 10 * 2