def test_suggest_low_equals_high(storage_init_func):
# type: (typing.Callable[[], storages.BaseStorage]) -> None
study = create_study(storage_init_func(),
sampler=samplers.TPESampler(n_startup_trials=0))
trial = Trial(study, study._storage.create_new_trial(study.study_id))
# Parameter values are determined without suggestion when low == high.
with patch.object(trial, '_suggest', wraps=trial._suggest) as mock_object:
assert trial.suggest_uniform('a', 1., 1.) == 1. # Suggesting a param.
assert trial.suggest_uniform('a', 1.,
1.) == 1. # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_loguniform('b', 1.,
1.) == 1. # Suggesting a param.
assert trial.suggest_loguniform('b', 1.,
1.) == 1. # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_discrete_uniform('c', 1., 1.,
1.) == 1. # Suggesting a param.
assert trial.suggest_discrete_uniform(
'c', 1., 1., 1.) == 1. # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_int('d', 1, 1) == 1 # Suggesting a param.
assert trial.suggest_int('d', 1, 1) == 1 # Suggesting the same param.
assert mock_object.call_count == 0
def obj(t: Trial) -> float:
t.suggest_uniform("a", 1.0, 100.0)
t.suggest_loguniform("b", 1.0, 100.0)
t.suggest_discrete_uniform("c", 1.0, 100.0, 3.0)
t.suggest_int("d", 1, 100)
t.suggest_int("e", 0, 100, step=2)
t.suggest_int("f", 1, 100, log=True)
t.suggest_categorical("g", ["x", "y", "z"])
return 0.0
def test_suggest_low_equals_high(storage_mode: str) -> None:
with patch.object(
distributions, "_get_single_value", wraps=distributions._get_single_value
) as mock_object, StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=samplers.TPESampler(n_startup_trials=0))
trial = Trial(study, study._storage.create_new_trial(study._study_id))
assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 1
assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 1
assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 2
assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 2
assert trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 3
assert (
trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0
) # Suggesting the same param.
assert mock_object.call_count == 3
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param.
assert mock_object.call_count == 4
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param.
assert mock_object.call_count == 4
assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 5
assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 5
assert trial.suggest_float("f", 0.5, 0.5, log=True) == 0.5 # Suggesting a param.
assert mock_object.call_count == 6
assert trial.suggest_float("f", 0.5, 0.5, log=True) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 6
assert trial.suggest_float("g", 0.5, 0.5, log=False) == 0.5 # Suggesting a param.
assert mock_object.call_count == 7
assert trial.suggest_float("g", 0.5, 0.5, log=False) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 7
assert trial.suggest_float("h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting a param.
assert mock_object.call_count == 8
assert trial.suggest_float("h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 8
assert trial.suggest_int("i", 1, 1, log=True) == 1 # Suggesting a param.
assert mock_object.call_count == 9
assert trial.suggest_int("i", 1, 1, log=True) == 1 # Suggesting the same param.
assert mock_object.call_count == 9
def sample_params_values(self, trial: Trial, suggested_params: Dict,
estimated_n_trials: int) -> Dict:
"""Sample hyperparameters from suggested.
Args:
trial: optuna trial object.
suggested_params: dict with parameters.
estimated_n_trials: maximum number of hyperparameter estimations.
Returns:
dict with sampled hyperparameters.
"""
logger.debug('Suggested parameters:')
logger.debug(suggested_params)
trial_values = copy(suggested_params)
trial_values['feature_fraction'] = trial.suggest_uniform(
name='feature_fraction',
low=0.5,
high=1.0,
)
trial_values['num_leaves'] = trial.suggest_int(
name='num_leaves',
low=16,
high=255,
)
if estimated_n_trials > 30:
trial_values['bagging_fraction'] = trial.suggest_uniform(
name='bagging_fraction',
low=0.5,
high=1.0,
)
trial_values['min_sum_hessian_in_leaf'] = trial.suggest_loguniform(
name='min_sum_hessian_in_leaf',
low=1e-3,
high=10.0,
)
if estimated_n_trials > 100:
trial_values['reg_alpha'] = trial.suggest_loguniform(
name='reg_alpha',
low=1e-8,
high=10.0,
)
trial_values['reg_lambda'] = trial.suggest_loguniform(
name='reg_lambda',
low=1e-8,
high=10.0,
)
return trial_values
def objective(trial: Trial) -> float:
trial.suggest_uniform("a", 0, 10)
trial.suggest_loguniform("b", 0.1, 10)
trial.suggest_discrete_uniform("c", 0, 10, 1)
trial.suggest_int("d", 0, 10)
trial.suggest_categorical("e", ["foo", "bar", "baz"])
trial.suggest_int("f", 1, 10, log=True)
return 1.0
def objective(trial: Trial) -> float:
if trial.number == 0:
trial.suggest_uniform("param1", 0, 1)
raise optuna.exceptions.TrialPruned()
if trial.number == 1:
trial.suggest_uniform("param2", 0, 1)
return 0
return 0
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
x2 = trial.suggest_loguniform("x2", 0.1, 3)
x3 = trial.suggest_discrete_uniform("x3", 0, 3, 1)
if trial.number % 2 == 0:
x4 = trial.suggest_uniform("x4", 0.1, 3)
value = x1**4 + x2 + x3
if trial.number % 2 == 0:
value += x4
return value
def objective(trial: Trial) -> float:
# Predefined parameters are sampled by `sample_relative()` method.
assert trial.suggest_uniform("a", 0, 5) == 3.2
assert trial.suggest_categorical("b", ["foo", "bar", "baz"]) == "baz"
# Other parameters are sampled by `sample_independent()` method.
assert trial.suggest_int("c", 20, 50) == unknown_param_value
assert trial.suggest_loguniform("d", 1, 100) == unknown_param_value
assert trial.suggest_uniform("e", 20, 40) == unknown_param_value
return 0.0
def test_check_distribution_suggest_uniform(storage_init_func):
# type: (typing.Callable[[], storages.BaseStorage]) -> None
sampler = samplers.RandomSampler()
study = create_study(storage_init_func(), sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.warns(None) as record:
trial.suggest_uniform("x", 10, 20)
trial.suggest_uniform("x", 10, 20)
trial.suggest_uniform("x", 10, 30)
# we expect exactly one warning
assert len(record) == 1
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
x2 = trial.suggest_loguniform("x2", 0.1, 3)
x3 = trial.suggest_discrete_uniform("x3", 0, 3, 1)
x4 = trial.suggest_int("x4", -3, 3)
x5 = trial.suggest_categorical("x5", [1.0, 1.1, 1.2])
if trial.number % 2 == 0:
# Conditional parameters are ignored unless `params` is specified and is not `None`.
x6 = trial.suggest_uniform("x6", 0.1, 3)
assert isinstance(x5, float)
value = x1**4 + x2 + x3 - x4**2 - x5
if trial.number % 2 == 0:
value += x6
return value
def test_suggest_uniform(storage_mode: str) -> None:
sampler = DeterministicSampler({"x": 1.0, "y": 2.0})
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
assert trial.suggest_uniform("x", 0.0,
3.0) == 1.0 # Test suggesting a param.
assert trial.suggest_uniform(
"x", 0.0, 3.0) == 1.0 # Test suggesting the same param.
assert trial.suggest_uniform(
"y", 0.0, 3.0) == 2.0 # Test suggesting a different param.
assert trial.params == {"x": 1.0, "y": 2.0}
def test_check_distribution_suggest_float(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
sampler = samplers.RandomSampler()
study = create_study(storage_init_func(), sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
x1 = trial.suggest_float("x1", 10, 20)
x2 = trial.suggest_uniform("x1", 10, 20)
assert x1 == x2
x3 = trial.suggest_float("x2", 1e-5, 1e-3, log=True)
x4 = trial.suggest_loguniform("x2", 1e-5, 1e-3)
assert x3 == x4
x5 = trial.suggest_float("x3", 10, 20, step=1.0)
x6 = trial.suggest_discrete_uniform("x3", 10, 20, 1.0)
assert x5 == x6
with pytest.raises(ValueError):
trial.suggest_float("x4", 1e-5, 1e-2, step=1e-5, log=True)
with pytest.raises(ValueError):
trial.suggest_int("x1", 10, 20)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.raises(ValueError):
trial.suggest_int("x1", 10, 20)
def test_suggest_low_equals_high(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
study = create_study(storage_init_func(), sampler=samplers.TPESampler(n_startup_trials=0))
trial = Trial(study, study._storage.create_new_trial(study._study_id))
# Parameter values are determined without suggestion when low == high.
with patch.object(trial, "_suggest", wraps=trial._suggest) as mock_object:
assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting a param.
assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting a param.
assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param.
assert (
trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0
) # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param.
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting a param.
assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_float("f", 0.5, 0.5, log=True) == 0.5 # Suggesting a param.
assert trial.suggest_float("f", 0.5, 0.5, log=True) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_float("g", 0.5, 0.5, log=False) == 0.5 # Suggesting a param.
assert trial.suggest_float("g", 0.5, 0.5, log=False) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 0
def test_check_distribution_suggest_float(storage_mode: str) -> None:
sampler = samplers.RandomSampler()
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
x1 = trial.suggest_float("x1", 10, 20)
x2 = trial.suggest_uniform("x1", 10, 20)
assert x1 == x2
x3 = trial.suggest_float("x2", 1e-5, 1e-3, log=True)
x4 = trial.suggest_loguniform("x2", 1e-5, 1e-3)
assert x3 == x4
x5 = trial.suggest_float("x3", 10, 20, step=1.0)
x6 = trial.suggest_discrete_uniform("x3", 10, 20, 1.0)
assert x5 == x6
with pytest.raises(ValueError):
trial.suggest_float("x4", 1e-5, 1e-2, step=1e-5, log=True)
with pytest.raises(ValueError):
trial.suggest_int("x1", 10, 20)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.raises(ValueError):
trial.suggest_int("x1", 10, 20)
def objective(trial: Trial) -> float:
a = trial.suggest_int("a", 0, 100)
b = trial.suggest_uniform("b", -0.1, 0.1)
c = trial.suggest_categorical("c", ("x", "y"))
d = trial.suggest_discrete_uniform("d", -5, 5, 1)
e = trial.suggest_loguniform("e", 0.0001, 1)
if c == "x":
return a * d
else:
return b * e
def get_optuna_parameter(self, trial: Trial):
if self.suggestion_type == OptunaSuggestion.DiscreteUniform:
return trial.suggest_discrete_uniform(self.name, self.low,
self.high, self.kwargs["q"])
elif self.suggestion_type == OptunaSuggestion.Int:
return trial.suggest_int(self.name, self.low, self.high)
elif self.suggestion_type == OptunaSuggestion.LogUniform:
return trial.suggest_loguniform(self.name, self.low, self.high)
elif self.suggestion_type == OptunaSuggestion.Uniform:
return trial.suggest_uniform(self.name, self.low, self.high)
else:
raise NotImplementedError
def test_check_distribution_suggest_uniform(storage_mode: str) -> None:
sampler = samplers.RandomSampler()
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.warns(None) as record:
trial.suggest_uniform("x", 10, 20)
trial.suggest_uniform("x", 10, 20)
trial.suggest_uniform("x", 10, 30)
# we expect exactly one warning
assert len(record) == 1
with pytest.raises(ValueError):
trial.suggest_int("x", 10, 20)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.raises(ValueError):
trial.suggest_int("x", 10, 20)
def generate_model_class_try_params(self, trial: Trial):
params = {
'weights': trial.suggest_categorical('weights', ['distance', 'uniform']),
'p': trial.suggest_uniform('p', 1, 4),
'n_neighbors': int(trial.suggest_int('n_neighbors', 5, 30)),
'algorithm': trial.suggest_categorical('algorithm', ['ball_tree', 'kd_tree'])
}
if 'tree' in params.get('algorithm', None):
params['leaf_size'] = int(trial.suggest_int('leaf_size', 10, 200))
return params
def test_check_distribution_suggest_uniform(storage_mode: str) -> None:
sampler = samplers.RandomSampler()
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.warns(None) as record:
trial.suggest_uniform("x", 10, 20)
trial.suggest_uniform("x", 10, 20)
trial.suggest_uniform("x", 10, 30)
# we expect exactly one warning (not counting ones caused by deprecation)
assert len([r for r in record if r.category != FutureWarning]) == 1
with pytest.raises(ValueError):
trial.suggest_int("x", 10, 20)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.raises(ValueError):
trial.suggest_int("x", 10, 20)
def objective(trial: Trial) -> Tuple[float, float]:
p0 = trial.suggest_float("p0", -10, 10)
p1 = trial.suggest_uniform("p1", 3, 5)
p2 = trial.suggest_loguniform("p2", 0.00001, 0.1)
p3 = trial.suggest_discrete_uniform("p3", 100, 200, q=5)
p4 = trial.suggest_int("p4", -20, -15)
p5 = cast(int, trial.suggest_categorical("p5", [7, 1, 100]))
p6 = trial.suggest_float("p6", -10, 10, step=1.0)
p7 = trial.suggest_int("p7", 1, 7, log=True)
return (
p0 + p1 + p2,
p3 + p4 + p5 + p6 + p7,
)
def test_check_distribution_suggest_float(storage_init_func):
# type: (typing.Callable[[], storages.BaseStorage]) -> None
sampler = samplers.RandomSampler()
study = create_study(storage_init_func(), sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
x1 = trial.suggest_float("x1", 10, 20)
x2 = trial.suggest_uniform("x1", 10, 20)
assert x1 == x2
x3 = trial.suggest_float("x2", 1e-5, 1e-3, log=True)
x4 = trial.suggest_loguniform("x2", 1e-5, 1e-3)
assert x3 == x4
def define_hyperparameters(trial: Trial) -> Pipeline:
ngram_range = trial.suggest_categorical("vectorizer__ngram_range",
["11", "12"])
vectorizer = TfidfVectorizer(
stop_words="english",
min_df=2,
ngram_range=string_to_tuple(ngram_range),
)
clf = trial.suggest_categorical(
"clf", ["SVC", "RandomForestClassifier", "MultinomialNB"])
if clf == "SVC":
C = trial.suggest_uniform("clf__C", 0.1, 0.2)
classifier = SVC(C=C)
elif clf == "RandomForestClassifier":
max_depth = trial.suggest_int("clf__max_depth", 2, 4)
classifier = RandomForestClassifier(max_depth=max_depth)
else:
alpha = trial.suggest_loguniform("clf__alpha", 1e-2, 1e-1)
classifier = MultinomialNB(alpha=alpha)
return Pipeline([("vectorizer", vectorizer), ("clf", classifier)])
def _get_params(self, trial: trial_module.Trial) -> Dict[str, Any]:
params = self.params.copy() # type: Dict[str, Any]
if self.param_distributions is None:
params["colsample_bylevel"] = trial.suggest_discrete_uniform(
"colsample_bylevel", 0.1, 1.0, 0.05
)
params["max_depth"] = trial.suggest_int("max_depth", 1, 7)
# https://catboost.ai/docs/concepts/parameter-tuning.html#tree-growing-policy
# params["num_leaves"] = trial.suggest_int(
# "num_leaves", 2, 2 ** params["max_depth"]
# )
# See https://github.com/Microsoft/LightGBM/issues/907
params["num_leaves"] = 31
params["min_data_in_leaf"] = trial.suggest_int(
"min_data_in_leaf",
1,
max(1, int(self.n_samples / params["num_leaves"])),
)
params["l2_leaf_reg"] = trial.suggest_loguniform("lambda_l2", 1e-09, 10.0)
if params["bootstrap_type"] == "Bayesian":
params["bagging_temperature"] = trial.suggest_discrete_uniform(
"bagging_temperature", 0.5, 0.95, 0.05
)
elif (
params["bootstrap_type"] == "Bernoulli"
or params["bootstrap_type"] == "Poisson"
):
params["subsample"] = trial.suggest_uniform("subsample", 0.1, 1)
return params
for name, distribution in self.param_distributions.items():
params[name] = trial._suggest(name, distribution)
return params
def objective_func(trial: Trial) -> float:
x = trial.suggest_uniform("x", -10, 10)
return (x + 5)**2
def objective(trial: Trial) -> float:
a = trial.suggest_int("a", 0, 100)
b = trial.suggest_uniform("b", -100, 100)
return a * b
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
return x1**2
def objective(trial: Trial, base_value: float) -> float:
return trial.suggest_uniform("x", 0.1, 0.2) + base_value
def objective(trial: Trial, exception: Exception) -> float:
trial.suggest_uniform("z", 0, 1)
raise exception
def __call__(self, name: str, trial: Trial):
return trial.suggest_uniform(name, self.low, self.high)
def test_suggest_low_equals_high(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
study = create_study(storage_init_func(),
sampler=samplers.TPESampler(n_startup_trials=0))
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with patch.object(
optuna.distributions,
"_get_single_value",
wraps=optuna.distributions._get_single_value) as mock_object:
assert trial.suggest_uniform("a", 1.0,
1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 1
assert trial.suggest_uniform("a", 1.0,
1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 1
assert trial.suggest_loguniform("b", 1.0,
1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 2
assert trial.suggest_loguniform(
"b", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 2
assert trial.suggest_discrete_uniform(
"c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 3
assert (trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0
) # Suggesting the same param.
assert mock_object.call_count == 3
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param.
assert mock_object.call_count == 4
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param.
assert mock_object.call_count == 4
assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 5
assert trial.suggest_float("e", 1.0,
1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 5
assert trial.suggest_float("f", 0.5, 0.5,
log=True) == 0.5 # Suggesting a param.
assert mock_object.call_count == 6
assert trial.suggest_float(
"f", 0.5, 0.5, log=True) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 6
assert trial.suggest_float("g", 0.5, 0.5,
log=False) == 0.5 # Suggesting a param.
assert mock_object.call_count == 7
assert trial.suggest_float(
"g", 0.5, 0.5, log=False) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 7
assert trial.suggest_float("h", 0.5, 0.5,
step=1.0) == 0.5 # Suggesting a param.
assert mock_object.call_count == 8
assert trial.suggest_float(
"h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 8
assert trial.suggest_int("i", 1, 1,
log=True) == 1 # Suggesting a param.
assert mock_object.call_count == 9
assert trial.suggest_int("i", 1, 1,
log=True) == 1 # Suggesting the same param.
assert mock_object.call_count == 9
