assert sampler._rng.bytes(10) == restored_sampler._rng.bytes(10) def test_random_sampler_reseed_rng() -> None: sampler = optuna.samplers.RandomSampler() original_seed = sampler._rng.seed sampler.reseed_rng() assert original_seed != sampler._rng.seed @parametrize_sampler @pytest.mark.parametrize( "distribution", [ UniformDistribution(-1.0, 1.0), UniformDistribution(0.0, 1.0), UniformDistribution(-1.0, 0.0), ], ) def test_uniform( sampler_class: Callable[[], BaseSampler], distribution: UniformDistribution ) -> None: study = optuna.study.create_study(sampler=sampler_class()) points = np.array( [ study.sampler.sample_independent(study, _create_new_trial(study), "x", distribution) for _ in range(100) ] )
def test_create_new_trial_with_template_trial(storage_mode: str) -> None: start_time = datetime.now() complete_time = datetime.now() template_trial = FrozenTrial( state=TrialState.COMPLETE, value=10000, datetime_start=start_time, datetime_complete=complete_time, params={"x": 0.5}, distributions={"x": UniformDistribution(0, 1)}, user_attrs={"foo": "bar"}, system_attrs={"baz": 123}, intermediate_values={ 1: 10, 2: 100, 3: 1000 }, number=55, # This entry is ignored. trial_id=-1, # dummy value (unused). ) def _check_trials(trials: List[FrozenTrial], idx: int, trial_id: int) -> None: assert len(trials) == idx + 1 assert len({t._trial_id for t in trials}) == idx + 1 assert trial_id in {t._trial_id for t in trials} assert {t.number for t in trials} == set(range(idx + 1)) assert all(t.state == template_trial.state for t in trials) assert all(t.params == template_trial.params for t in trials) assert all(t.distributions == template_trial.distributions for t in trials) assert all(t.intermediate_values == template_trial.intermediate_values for t in trials) assert all(t.user_attrs == template_trial.user_attrs for t in trials) assert all(t.system_attrs == template_trial.system_attrs for t in trials) assert all(t.datetime_start == template_trial.datetime_start for t in trials) assert all(t.datetime_complete == template_trial.datetime_complete for t in trials) assert all(t.value == template_trial.value for t in trials) with StorageSupplier(storage_mode) as storage: study_id = storage.create_new_study() n_trial_in_study = 3 for i in range(n_trial_in_study): trial_id = storage.create_new_trial(study_id, template_trial=template_trial) trials = storage.get_all_trials(study_id) _check_trials(trials, i, trial_id) # Create trial in non-existent study. with pytest.raises(KeyError): storage.create_new_trial(study_id + 1) study_id2 = storage.create_new_study() for i in range(n_trial_in_study): storage.create_new_trial(study_id2, template_trial=template_trial) trials = storage.get_all_trials(study_id2) assert {t.number for t in trials} == set(range(i + 1)) trials = storage.get_all_trials(study_id) + storage.get_all_trials( study_id2) # Check trial_ids are unique across studies. assert len({t._trial_id for t in trials}) == 2 * n_trial_in_study
def prepare_study_with_trials( no_trials: bool = False, less_than_two: bool = False, more_than_three: bool = False, with_c_d: bool = True, ) -> Study: """Prepare a study for tests. Args: no_trials: If ``False``, create a study with no trials. less_than_two: If ``True``, create a study with two/four hyperparameters where 'param_a' (and 'param_c') appear(s) only once while 'param_b' (and 'param_d') appear(s) twice in `study.trials`. more_than_three: If ``True``, create a study with two/four hyperparameters where 'param_a' (and 'param_c') appear(s) only three times while 'param_b' (and 'param_d') appear(s) four times in `study.trials`. with_c_d: If ``True``, the study has four hyperparameters named 'param_a', 'param_b', 'param_c', and 'param_d'. Otherwise, there are only two hyperparameters ('param_a' and 'param_b'). Returns: :class:`~optuna.study.Study` """ study = create_study() if no_trials: return study study.add_trial( create_trial( value=0.0, params={ "param_a": 1.0, "param_b": 2.0, "param_c": 3.0, "param_d": 4.0 } if with_c_d else { "param_a": 1.0, "param_b": 2.0 }, distributions={ "param_a": UniformDistribution(0.0, 3.0), "param_b": UniformDistribution(0.0, 3.0), "param_c": UniformDistribution(2.0, 5.0), "param_d": UniformDistribution(2.0, 5.0), } if with_c_d else { "param_a": UniformDistribution(0.0, 3.0), "param_b": UniformDistribution(0.0, 3.0), }, )) study.add_trial( create_trial( value=2.0, params={ "param_b": 0.0, "param_d": 4.0 } if with_c_d else {"param_b": 0.0}, distributions={ "param_b": UniformDistribution(0.0, 3.0), "param_d": UniformDistribution(2.0, 5.0), } if with_c_d else {"param_b": UniformDistribution(0.0, 3.0)}, )) if less_than_two: return study study.add_trial( create_trial( value=1.0, params={ "param_a": 2.5, "param_b": 1.0, "param_c": 4.5, "param_d": 2.0 } if with_c_d else { "param_a": 2.5, "param_b": 1.0 }, distributions={ "param_a": UniformDistribution(0.0, 3.0), "param_b": UniformDistribution(0.0, 3.0), "param_c": UniformDistribution(2.0, 5.0), "param_d": UniformDistribution(2.0, 5.0), } if with_c_d else { "param_a": UniformDistribution(0.0, 3.0), "param_b": UniformDistribution(0.0, 3.0), }, )) if more_than_three: study.add_trial( create_trial( value=1.5, params={ "param_a": 0.5, "param_b": 1.5, "param_c": 2.0, "param_d": 3.0 } if with_c_d else { "param_a": 0.5, "param_b": 1.5 }, distributions={ "param_a": UniformDistribution(0.0, 3.0), "param_b": UniformDistribution(0.0, 3.0), "param_c": UniformDistribution(2.0, 5.0), "param_d": UniformDistribution(2.0, 5.0), } if with_c_d else { "param_a": UniformDistribution(0.0, 3.0), "param_b": UniformDistribution(0.0, 3.0), }, )) return study
from typing import Optional # NOQA from typing import Tuple # NOQA # TODO(Yanase): Remove _number from system_attrs after adding TrialModel.number. EXAMPLE_ATTRS = { 'dataset': 'MNIST', 'none': None, 'json_serializable': { 'baseline_score': 0.001, 'tags': ['image', 'classification'] }, '_number': 0, } EXAMPLE_DISTRIBUTIONS = { 'x': UniformDistribution(low=1., high=2.), 'y': CategoricalDistribution(choices=('Otemachi', 'Tokyo', 'Ginza')) } # type: Dict[str, BaseDistribution] # TODO(Yanase): Remove _number from system_attrs after adding TrialModel.number. EXAMPLE_TRIALS = [ FrozenTrial( number=0, # dummy value=1., state=TrialState.COMPLETE, user_attrs={}, system_attrs={'_number': 0}, params={ 'x': 0.5, 'y': 'Ginza' },
def test_frozen_trial_validate(): # type: () -> None # Valid. valid_trial = _create_frozen_trial() valid_trial._validate() # Invalid: `datetime_start` is not set. invalid_trial = copy.copy(valid_trial) invalid_trial.datetime_start = None with pytest.raises(ValueError): invalid_trial._validate() # Invalid: `state` is `RUNNING` and `datetime_complete` is set. invalid_trial = copy.copy(valid_trial) invalid_trial.state = TrialState.RUNNING with pytest.raises(ValueError): invalid_trial._validate() # Invalid: `state` is not `RUNNING` and `datetime_complete` is not set. for state in [TrialState.COMPLETE, TrialState.PRUNED, TrialState.FAIL]: invalid_trial = copy.copy(valid_trial) invalid_trial.state = state invalid_trial.datetime_complete = None with pytest.raises(ValueError): invalid_trial._validate() # Invalid: `state` is `COMPLETE` and `value` is not set. invalid_trial = copy.copy(valid_trial) invalid_trial.value = None with pytest.raises(ValueError): invalid_trial._validate() # Invalid: Inconsistent `params` and `distributions` inconsistent_pairs = [ # `params` has an extra element. ({ 'x': 0.1, 'y': 0.5 }, { 'x': UniformDistribution(0, 1) }), # `distributions` has an extra element. ({ 'x': 0.1 }, { 'x': UniformDistribution(0, 1), 'y': LogUniformDistribution(0, 1) }), # The value of `x` isn't contained in the distribution. ({ 'x': -0.5 }, { 'x': UniformDistribution(0, 1) }) ] # type: List[Tuple[Dict[str, Any], Dict[str, BaseDistribution]]] for params, distributions in inconsistent_pairs: invalid_trial = copy.copy(valid_trial) invalid_trial.params = params invalid_trial.distributions = distributions with pytest.raises(ValueError): invalid_trial._validate()
def test_set_trial_param(storage_mode: str) -> None: with StorageSupplier(storage_mode) as storage: # Setup test across multiple studies and trials. study_id = storage.create_new_study() trial_id_1 = storage.create_new_trial(study_id) trial_id_2 = storage.create_new_trial(study_id) trial_id_3 = storage.create_new_trial(storage.create_new_study()) # Setup distributions. distribution_x = UniformDistribution(low=1.0, high=2.0) distribution_y_1 = CategoricalDistribution(choices=("Shibuya", "Ebisu", "Meguro")) distribution_y_2 = CategoricalDistribution(choices=("Shibuya", "Shinsen")) distribution_z = LogUniformDistribution(low=1.0, high=100.0) # Set new params. storage.set_trial_param(trial_id_1, "x", 0.5, distribution_x) storage.set_trial_param(trial_id_1, "y", 2, distribution_y_1) assert storage.get_trial_param(trial_id_1, "x") == 0.5 assert storage.get_trial_param(trial_id_1, "y") == 2 # Check set_param breaks neither get_trial nor get_trial_params. assert storage.get_trial(trial_id_1).params == { "x": 0.5, "y": "Meguro" } assert storage.get_trial_params(trial_id_1) == { "x": 0.5, "y": "Meguro" } # Duplicated registration should overwrite. storage.set_trial_param(trial_id_1, "x", 0.6, distribution_x) assert storage.get_trial_param(trial_id_1, "x") == 0.6 assert storage.get_trial(trial_id_1).params == { "x": 0.6, "y": "Meguro" } assert storage.get_trial_params(trial_id_1) == { "x": 0.6, "y": "Meguro" } # Set params to another trial. storage.set_trial_param(trial_id_2, "x", 0.3, distribution_x) storage.set_trial_param(trial_id_2, "z", 0.1, distribution_z) assert storage.get_trial_param(trial_id_2, "x") == 0.3 assert storage.get_trial_param(trial_id_2, "z") == 0.1 assert storage.get_trial(trial_id_2).params == {"x": 0.3, "z": 0.1} assert storage.get_trial_params(trial_id_2) == {"x": 0.3, "z": 0.1} # Set params with distributions that do not match previous ones. with pytest.raises(ValueError): storage.set_trial_param(trial_id_2, "x", 0.5, distribution_z) with pytest.raises(ValueError): storage.set_trial_param(trial_id_2, "y", 0.5, distribution_z) # Choices in CategoricalDistribution should match including its order. with pytest.raises(ValueError): storage.set_trial_param( trial_id_2, "y", 2, CategoricalDistribution(choices=("Meguro", "Shibuya", "Ebisu"))) storage.set_trial_state(trial_id_2, TrialState.COMPLETE) # Cannot assign params to finished trial. with pytest.raises(RuntimeError): storage.set_trial_param(trial_id_2, "y", 2, distribution_y_1) # Check the previous call does not change the params. with pytest.raises(KeyError): storage.get_trial_param(trial_id_2, "y") # State should be checked prior to distribution compatibility. with pytest.raises(RuntimeError): storage.set_trial_param(trial_id_2, "y", 0.4, distribution_z) # Set params of trials in a different study. storage.set_trial_param(trial_id_3, "y", 1, distribution_y_2) assert storage.get_trial_param(trial_id_3, "y") == 1 assert storage.get_trial(trial_id_3).params == {"y": "Shinsen"} assert storage.get_trial_params(trial_id_3) == {"y": "Shinsen"} # Set params of non-existent trial. non_existent_trial_id = max([trial_id_1, trial_id_2, trial_id_3]) + 1 with pytest.raises(KeyError): storage.set_trial_param(non_existent_trial_id, "x", 0.1, distribution_x)
from optuna.trial import Trial # NOQA parametrize_sampler = pytest.mark.parametrize( 'sampler_class', [ optuna.samplers.RandomSampler, lambda: optuna.samplers.TPESampler(n_startup_trials=0), lambda: optuna.integration.SkoptSampler(skopt_kwargs={'n_initial_points': 1}), lambda: optuna.integration.CmaEsSampler() ]) @parametrize_sampler @pytest.mark.parametrize( 'distribution', [UniformDistribution(-1., 1.), UniformDistribution(0., 1.), UniformDistribution(-1., 0.)]) def test_uniform(sampler_class, distribution): # type: (typing.Callable[[], BaseSampler], UniformDistribution) -> None study = optuna.study.create_study(sampler=sampler_class()) points = np.array([ study.sampler.sample_independent(study, _create_new_trial(study), 'x', distribution) for _ in range(100) ]) assert np.all(points >= distribution.low) assert np.all(points < distribution.high) assert not isinstance( study.sampler.sample_independent(study, _create_new_trial(study), 'x', distribution), np.floating)
def suggest_uniform(self, name: str, low: float, high: float) -> float: return self._suggest(name, UniformDistribution(low=low, high=high))
def test_intersection_search_space() -> None: search_space = optuna.samplers.IntersectionSearchSpace() study = optuna.create_study() # No trial. assert search_space.calculate(study) == {} assert search_space.calculate( study) == optuna.samplers.intersection_search_space(study) # First trial. study.optimize( lambda t: t.suggest_uniform("y", -3, 3) + t.suggest_int("x", 0, 10), n_trials=1) assert search_space.calculate(study) == { "x": IntUniformDistribution(low=0, high=10), "y": UniformDistribution(low=-3, high=3), } assert search_space.calculate( study) == optuna.samplers.intersection_search_space(study) # Returning sorted `OrderedDict` instead of `dict`. assert search_space.calculate(study, ordered_dict=True) == OrderedDict([ ("x", IntUniformDistribution(low=0, high=10)), ("y", UniformDistribution(low=-3, high=3)), ]) assert search_space.calculate( study, ordered_dict=True) == optuna.samplers.intersection_search_space( study, ordered_dict=True) # Second trial (only 'y' parameter is suggested in this trial). study.optimize(lambda t: t.suggest_uniform("y", -3, 3), n_trials=1) assert search_space.calculate(study) == { "y": UniformDistribution(low=-3, high=3) } assert search_space.calculate( study) == optuna.samplers.intersection_search_space(study) # Failed or pruned trials are not considered in the calculation of # an intersection search space. def objective(trial, exception): # type: (optuna.trial.Trial, Exception) -> float trial.suggest_uniform("z", 0, 1) raise exception study.optimize(lambda t: objective(t, RuntimeError()), n_trials=1, catch=(RuntimeError, )) study.optimize(lambda t: objective(t, optuna.exceptions.TrialPruned()), n_trials=1) assert search_space.calculate(study) == { "y": UniformDistribution(low=-3, high=3) } assert search_space.calculate( study) == optuna.samplers.intersection_search_space(study) # If two parameters have the same name but different distributions, # those are regarded as different parameters. study.optimize(lambda t: t.suggest_uniform("y", -1, 1), n_trials=1) assert search_space.calculate(study) == {} assert search_space.calculate( study) == optuna.samplers.intersection_search_space(study) # The search space remains empty once it is empty. study.optimize( lambda t: t.suggest_uniform("y", -3, 3) + t.suggest_int("x", 0, 10), n_trials=1) assert search_space.calculate(study) == {} assert search_space.calculate( study) == optuna.samplers.intersection_search_space(study)