def _create_new_chainermn_trial(study, comm): # type: (Study, CommunicatorBase) -> integration.chainermn.ChainerMNTrial if comm.rank == 0: trial_id = study._storage.create_new_trial(study.study_id) trial = Trial(study, trial_id) mn_trial = integration.chainermn.ChainerMNTrial(trial, comm) else: mn_trial = integration.chainermn.ChainerMNTrial(None, comm) comm.mpi_comm.barrier() return mn_trial
def test_check_distribution_suggest_loguniform( storage_init_func: Callable[[], storages.BaseStorage]) -> None: sampler = samplers.RandomSampler() study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(None) as record: trial.suggest_loguniform("x", 10, 20) trial.suggest_loguniform("x", 10, 20) trial.suggest_loguniform("x", 10, 30) # we expect exactly one warning assert len(record) == 1 with pytest.raises(ValueError): trial.suggest_int("x", 10, 20) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.raises(ValueError): trial.suggest_int("x", 10, 20)
def test_study_id(): # type: () -> None study = create_study() trial = Trial(study, study._storage.create_new_trial(study._study_id)) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) assert trial.study_id == trial.study._study_id with pytest.warns(DeprecationWarning): trial.study_id
def test_check_distribution_suggest_loguniform(storage_mode: str) -> None: sampler = samplers.RandomSampler() with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(None) as record: trial.suggest_loguniform("x", 10, 20) trial.suggest_loguniform("x", 10, 20) trial.suggest_loguniform("x", 10, 30) # we expect exactly one warning (not counting ones caused by deprecation) assert len([r for r in record if r.category != FutureWarning]) == 1 with pytest.raises(ValueError): trial.suggest_int("x", 10, 20) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.raises(ValueError): trial.suggest_int("x", 10, 20)
def test_report_and_should_prune(storage_mode, cache_mode, comm, is_pruning): # type: (str, bool, CommunicatorBase, bool) -> None with MultiNodeStorageSupplier(storage_mode, cache_mode, comm) as storage: study = TestChainerMNStudy._create_shared_study( storage, comm, DeterministicPruner(is_pruning)) trial_id = storage.create_new_trial_id(study.study_id) trial = Trial(study, trial_id) mn_trial = integration.chainermn._ChainerMNTrial(trial, comm) mn_trial.report(1.0, 0) assert mn_trial.should_prune(0) == is_pruning
def test_init(storage_mode, cache_mode, comm): # type: (str, bool, CommunicatorBase) -> None with MultiNodeStorageSupplier(storage_mode, cache_mode, comm) as storage: study = TestChainerMNStudy._create_shared_study(storage, comm) trial_id = storage.create_new_trial_id(study.study_id) trial = Trial(study, trial_id) mn_trial = integration.chainermn._ChainerMNTrial(trial, comm) assert mn_trial.trial_id == trial.trial_id assert mn_trial._trial_id == trial._trial_id assert mn_trial.number == trial.number
def test_check_distribution_suggest_discrete_uniform( storage_mode: str) -> None: sampler = samplers.RandomSampler() with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(None) as record: trial.suggest_discrete_uniform("x", 10, 20, 2) trial.suggest_discrete_uniform("x", 10, 20, 2) trial.suggest_discrete_uniform("x", 10, 22, 2) # we expect exactly one warning assert len(record) == 1 with pytest.raises(ValueError): trial.suggest_int("x", 10, 20, 2) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.raises(ValueError): trial.suggest_int("x", 10, 20, 2)
def test_check_distribution_suggest_int(storage_mode: str, enable_log: bool) -> None: sampler = samplers.RandomSampler() with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(None) as record: trial.suggest_int("x", 10, 20, log=enable_log) trial.suggest_int("x", 10, 20, log=enable_log) trial.suggest_int("x", 10, 22, log=enable_log) # We expect exactly one warning. assert len(record) == 1 with pytest.raises(ValueError): trial.suggest_float("x", 10, 20, log=enable_log) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.raises(ValueError): trial.suggest_float("x", 10, 20, log=enable_log)
def test_suggest_int_range(storage_mode: str, range_config: Dict[str, int]) -> None: sampler = samplers.RandomSampler() # Check upper endpoints. mock = Mock() mock.side_effect = lambda study, trial, param_name, distribution: distribution.high with patch.object( sampler, "sample_independent", mock) as mock_object, StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(UserWarning): x = trial.suggest_int("x", range_config["low"], range_config["high"], step=range_config["step"]) assert x == range_config["mod_high"] assert mock_object.call_count == 1 # Check lower endpoints. mock = Mock() mock.side_effect = lambda study, trial, param_name, distribution: distribution.low with patch.object( sampler, "sample_independent", mock) as mock_object, StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(UserWarning): x = trial.suggest_int("x", range_config["low"], range_config["high"], step=range_config["step"]) assert x == range_config["low"] assert mock_object.call_count == 1
def test_check_distribution_suggest_discrete_uniform(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None sampler = samplers.RandomSampler() study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(None) as record: trial.suggest_discrete_uniform("x", 10, 20, 2) trial.suggest_discrete_uniform("x", 10, 20, 2) trial.suggest_discrete_uniform("x", 10, 22, 2) # we expect exactly one warning assert len(record) == 1
def test_suggest_int(storage_mode: str) -> None: sampler = DeterministicSampler({"x": 1, "y": 2}) with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) assert trial.suggest_int("x", 0, 3) == 1 # Test suggesting a param. assert trial.suggest_int("x", 0, 3) == 1 # Test suggesting the same param. assert trial.suggest_int("y", 0, 3) == 2 # Test suggesting a different param. assert trial.params == {"x": 1, "y": 2}
def test_distributions(storage_mode, cache_mode, comm): # type: (str, bool, CommunicatorBase) -> None with MultiNodeStorageSupplier(storage_mode, cache_mode, comm) as storage: study = TestChainerMNStudy._create_shared_study(storage, comm) trial_id = storage.create_new_trial_id(study.study_id) trial = Trial(study, trial_id) mn_trial = integration.chainermn._ChainerMNTrial(trial, comm) mn_trial.suggest_categorical('x', [1]) assert mn_trial.distributions == { 'x': distributions.CategoricalDistribution(choices=(1, )) }
def test_suggest_discrete_uniform(storage_mode: str) -> None: sampler = DeterministicSampler({"x": 1.0, "y": 2.0}) with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) assert (trial.suggest_discrete_uniform("x", 0.0, 3.0, 1.0) == 1.0 ) # Test suggesting a param. assert (trial.suggest_discrete_uniform("x", 0.0, 3.0, 1.0) == 1.0 ) # Test suggesting the same param. assert (trial.suggest_discrete_uniform("y", 0.0, 3.0, 1.0) == 2.0 ) # Test suggesting a different param. assert trial.params == {"x": 1.0, "y": 2.0}
def test_check_distribution_suggest_int( storage_init_func: Callable[[], storages.BaseStorage], enable_log: bool ) -> None: sampler = samplers.RandomSampler() study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.warns(None) as record: trial.suggest_int("x", 10, 20, log=enable_log) trial.suggest_int("x", 10, 20, log=enable_log) trial.suggest_int("x", 10, 22, log=enable_log) # We expect exactly one warning. assert len(record) == 1
def test_suggest_int_log(storage_mode: str) -> None: mock = Mock() mock.side_effect = [1, 2] sampler = samplers.RandomSampler() with patch.object( sampler, "sample_independent", mock) as mock_object, StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) distribution = IntLogUniformDistribution(low=1, high=3) assert trial._suggest("x", distribution) == 1 # Test suggesting a param. assert trial._suggest( "x", distribution) == 1 # Test suggesting the same param. assert trial._suggest( "y", distribution) == 2 # Test suggesting a different param. assert trial.params == {"x": 1, "y": 2} assert mock_object.call_count == 2 with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with warnings.catch_warnings(): # UserWarning will be raised since [0.5, 10] is not divisible by 1. warnings.simplefilter("ignore", category=UserWarning) with pytest.raises(ValueError): trial.suggest_int("z", 0.5, 10, log=True) # type: ignore with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) with pytest.raises(ValueError): trial.suggest_int("w", 1, 3, step=2, log=True)
def test_check_distribution_suggest_float(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None sampler = samplers.RandomSampler() study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) x1 = trial.suggest_float("x1", 10, 20) x2 = trial.suggest_uniform("x1", 10, 20) assert x1 == x2 x3 = trial.suggest_float("x2", 1e-5, 1e-3, log=True) x4 = trial.suggest_loguniform("x2", 1e-5, 1e-3) assert x3 == x4
def test_suggest_int(storage_init_func: Callable[[], storages.BaseStorage]) -> None: mock = Mock() mock.side_effect = [1, 2] sampler = samplers.RandomSampler() with patch.object(sampler, "sample_independent", mock) as mock_object: study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) distribution = IntUniformDistribution(low=0, high=3) assert trial._suggest("x", distribution) == 1 # Test suggesting a param. assert trial._suggest("x", distribution) == 1 # Test suggesting the same param. assert trial._suggest("y", distribution) == 2 # Test suggesting a different param. assert trial.params == {"x": 1, "y": 2} assert mock_object.call_count == 2
def test_suggest_discrete_uniform(storage_init_func): # type: (Callable[[], storages.BaseStorage]) -> None mock = Mock() mock.side_effect = [1.0, 2.0, 3.0] sampler = samplers.RandomSampler() with patch.object(sampler, "sample_independent", mock) as mock_object: study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) distribution = DiscreteUniformDistribution(low=0.0, high=3.0, q=1.0) assert trial._suggest("x", distribution) == 1.0 # Test suggesting a param. assert trial._suggest("x", distribution) == 1.0 # Test suggesting the same param. assert trial._suggest("y", distribution) == 3.0 # Test suggesting a different param. assert trial.params == {"x": 1.0, "y": 3.0} assert mock_object.call_count == 3
def test_suggest_discrete_uniform(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None mock = Mock() mock.side_effect = [1., 2., 3.] sampler = samplers.RandomSampler() with patch.object(sampler, 'sample_independent', mock) as mock_object: study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study.storage.create_new_trial_id(study.study_id)) distribution = distributions.DiscreteUniformDistribution(low=0., high=3., q=1.) assert trial._suggest('x', distribution) == 1. # Test suggesting a param. assert trial._suggest('x', distribution) == 1. # Test suggesting the same param. assert trial._suggest('y', distribution) == 3. # Test suggesting a different param. assert trial.params == {'x': 1., 'y': 3.} assert mock_object.call_count == 3
def test_suggest_low_equals_high(storage_init_func): # type: (Callable[[], storages.BaseStorage]) -> None study = create_study(storage_init_func(), sampler=samplers.TPESampler(n_startup_trials=0)) trial = Trial(study, study._storage.create_new_trial(study._study_id)) # Parameter values are determined without suggestion when low == high. with patch.object(trial, "_suggest", wraps=trial._suggest) as mock_object: assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting a param. assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting a param. assert trial.suggest_loguniform( "b", 1.0, 1.0) == 1.0 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_discrete_uniform( "c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param. assert (trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0 ) # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param. assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting a param. assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("f", 0.5, 0.5, log=True) == 0.5 # Suggesting a param. assert trial.suggest_float( "f", 0.5, 0.5, log=True) == 0.5 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("g", 0.5, 0.5, log=False) == 0.5 # Suggesting a param. assert trial.suggest_float( "g", 0.5, 0.5, log=False) == 0.5 # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_float("h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting a param. assert trial.suggest_float( "h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting the same param. assert mock_object.call_count == 0
def optimize( self, func, # type: Callable[[Trial, CommunicatorBase], float] n_trials=None, # type: Optional[int] timeout=None, # type: Optional[float] catch=( Exception, ), # type: Union[Tuple[()], Tuple[Type[Exception]]] ): # type: (...) -> None """Optimize an objective function. This method provides the same interface as :func:`optuna.study.Study.optimize` except the absence of ``n_jobs`` argument. """ if self.comm.rank == 0: func_mn = _ChainerMNObjectiveFunc(func, self.comm) self.delegate.optimize(func_mn, n_trials=n_trials, timeout=timeout, catch=catch) self.comm.mpi_comm.bcast((False, None)) else: while True: has_next_trial, trial_id = self.comm.mpi_comm.bcast(None) if not has_next_trial: break trial = Trial(self.delegate, trial_id) try: func(_ChainerMNTrial(trial, self.comm), self.comm) # We assume that if a node raises an exception, # all other nodes will do the same. # # The responsibility to handle acceptable exceptions (i.e., `TrialPruned` and # `catch`) is in the rank-0 node, so other nodes simply ignore them. except TrialPruned: pass except catch: pass finally: # The following line mitigates memory problems that can be occurred in some # environments (e.g., services that use computing containers such as CircleCI). # Please refer to the following PR for further details: # https://github.com/pfnet/optuna/pull/325. gc.collect()
def test_suggest_discrete_uniform(storage_mode: str) -> None: mock = Mock() mock.side_effect = [1.0, 2.0] sampler = samplers.RandomSampler() with patch.object(sampler, "sample_independent", mock) as mock_object, StorageSupplier( storage_mode ) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) distribution = DiscreteUniformDistribution(low=0.0, high=3.0, q=1.0) assert trial._suggest("x", distribution) == 1.0 # Test suggesting a param. assert trial._suggest("x", distribution) == 1.0 # Test suggesting the same param. assert trial._suggest("y", distribution) == 2.0 # Test suggesting a different param. assert trial.params == {"x": 1.0, "y": 2.0} assert mock_object.call_count == 2
def test_trial_should_prune(): # type: () -> None study_id = 1 trial_id = 1 study_mock = MagicMock() study_mock.study_id = study_id study_mock.storage.get_trial.return_value.\ intermediate_values.keys.return_value = [1, 2, 3, 4, 5] study_mock.pruner.prune.return_value = True trial = Trial(study_mock, trial_id) # type: ignore trial.should_prune() study_mock.storage.get_trial.assert_called_once_with(trial_id) study_mock.pruner.prune.assert_called_once_with( study_mock.storage, study_id, trial_id, 5, )
def test_suggest_categorical(storage_mode, cache_mode, comm): # type: (str, bool, CommunicatorBase) -> None with MultiNodeStorageSupplier(storage_mode, cache_mode, comm) as storage: study = TestChainerMNStudy._create_shared_study(storage, comm) choices = ('a', 'b', 'c') for _ in range(10): trial_id = storage.create_new_trial_id(study.study_id) trial = Trial(study, trial_id) mn_trial = integration.chainermn._ChainerMNTrial(trial, comm) x1 = mn_trial.suggest_categorical('x', choices) assert x1 in choices x2 = mn_trial.suggest_categorical('x', choices) assert x1 == x2 with pytest.raises(ValueError): mn_trial.suggest_uniform('x', 0., 1.)
def test_suggest_low_equals_high(storage_init_func): # type: (typing.Callable[[], storages.BaseStorage]) -> None study = create_study(storage_init_func(), sampler=samplers.TPESampler(n_startup_trials=0)) trial = Trial(study, study.storage.create_new_trial_id(study.study_id)) # Parameter values are determined without suggestion when low == high. with patch.object(trial, '_suggest', wraps=trial._suggest) as mock_object: assert trial.suggest_uniform('a', 1., 1.) == 1. # Suggesting a param. assert trial.suggest_uniform('a', 1., 1.) == 1. # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_loguniform('b', 1., 1.) == 1. # Suggesting a param. assert trial.suggest_loguniform('b', 1., 1.) == 1. # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_discrete_uniform('c', 1., 1., 1.) == 1. # Suggesting a param. assert trial.suggest_discrete_uniform('c', 1., 1., 1.) == 1. # Suggesting the same param. assert mock_object.call_count == 0 assert trial.suggest_int('d', 1, 1) == 1 # Suggesting a param. assert trial.suggest_int('d', 1, 1) == 1 # Suggesting the same param. assert mock_object.call_count == 0
def test_suggest_int(storage_mode, cache_mode, comm): # type: (str, bool, CommunicatorBase) -> None with MultiNodeStorageSupplier(storage_mode, cache_mode, comm) as storage: study = TestChainerMNStudy._create_shared_study(storage, comm) low = 0 high = 10 for _ in range(10): trial_id = storage.create_new_trial_id(study.study_id) trial = Trial(study, trial_id) mn_trial = integration.chainermn._ChainerMNTrial(trial, comm) x1 = mn_trial.suggest_int('x', low, high) assert low <= x1 <= high x2 = mn_trial.suggest_int('x', low, high) assert x1 == x2 with pytest.raises(ValueError): mn_trial.suggest_uniform('x', low, high)
def test_suggest_loguniform(storage_mode: str) -> None: with pytest.raises(ValueError): FloatDistribution(low=1.0, high=0.9, log=True) with pytest.raises(ValueError): FloatDistribution(low=0.0, high=0.9, log=True) sampler = DeterministicSampler({"x": 1.0, "y": 2.0}) with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) assert trial.suggest_loguniform("x", 0.1, 4.0) == 1.0 # Test suggesting a param. assert trial.suggest_loguniform( "x", 0.1, 4.0) == 1.0 # Test suggesting the same param. assert trial.suggest_loguniform( "y", 0.1, 4.0) == 2.0 # Test suggesting a different param. assert trial.params == {"x": 1.0, "y": 2.0}
def optimize( self, func, # type: Callable[[Trial, CommunicatorBase], float] n_trials=None, # type: Optional[int] timeout=None, # type: Optional[float] catch=( Exception, ), # type: Union[Tuple[()], Tuple[Type[Exception]]] ): # type: (...) -> None """Optimize an objective function. This method provides the same interface as :func:`optuna.study.Study.optimize` except the absence of ``n_jobs`` argument. """ if self.comm.rank == 0: func_mn = _ChainerMNObjectiveFunc(func, self.comm) self.delegate.optimize(func_mn, n_trials=n_trials, timeout=timeout, catch=catch) self.comm.mpi_comm.bcast((False, None)) else: while True: has_next_trial, trial_id = self.comm.mpi_comm.bcast(None) if not has_next_trial: break trial = Trial(self.delegate, trial_id) try: func(_ChainerMNTrial(trial, self.comm), self.comm) # We assume that if a node raises an exception, # all other nodes will do the same. # # The responsibility to handle acceptable exceptions (i.e., `TrialPruned` and # `catch`) is in the rank-0 node, so other nodes simply ignore them. except TrialPruned: pass except catch: pass
def test_suggest_loguniform(storage_init_func: Callable[[], storages.BaseStorage]) -> None: with pytest.raises(ValueError): LogUniformDistribution(low=1.0, high=0.9) with pytest.raises(ValueError): LogUniformDistribution(low=0.0, high=0.9) mock = Mock() mock.side_effect = [1.0, 2.0] sampler = samplers.RandomSampler() with patch.object(sampler, "sample_independent", mock) as mock_object: study = create_study(storage_init_func(), sampler=sampler) trial = Trial(study, study._storage.create_new_trial(study._study_id)) distribution = LogUniformDistribution(low=0.1, high=4.0) assert trial._suggest("x", distribution) == 1.0 # Test suggesting a param. assert trial._suggest("x", distribution) == 1.0 # Test suggesting the same param. assert trial._suggest("y", distribution) == 2.0 # Test suggesting a different param. assert trial.params == {"x": 1.0, "y": 2.0} assert mock_object.call_count == 2
def test_trial_report(): # type: () -> None study = create_study() trial = Trial(study, study._storage.create_new_trial(study.study_id)) # Report values that can be cast to `float` (OK). trial.report(1.23) trial.report(float('nan')) trial.report('1.23') # type: ignore trial.report('inf') # type: ignore trial.report(1) trial.report(np.array([1], dtype=np.float32)[0]) # Report values that cannot be cast to `float` (Error). with pytest.raises(TypeError): trial.report(None) # type: ignore with pytest.raises(TypeError): trial.report('foo') # type: ignore with pytest.raises(TypeError): trial.report([1, 2, 3]) # type: ignore