def test_sample_independent_handle_unsuccessful_states(
state: optuna.trial.TrialState) -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
# Prepare sampling result for later tests.
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 30)]
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials):
all_success_suggestion = sampler.sample_independent(
study, trial, "param-a", dist)
# Test unsuccessful trials are handled differently.
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(i, dist=dist, state_fn=state_fn)
for i in range(1, 30)
]
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials):
partial_unsuccessful_suggestion = sampler.sample_independent(
study, trial, "param-a", dist)
assert partial_unsuccessful_suggestion != all_success_suggestion
def test_multi_objective_sample_independent_handle_unsuccessful_states(
state: optuna.trial.TrialState, ) -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
random.seed(128)
# Prepare sampling result for later tests.
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()]) for i in range(32)
]
trial = frozen_trial_factory(32, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
all_success_suggestion = sampler.sample_independent(
study, trial, "param-a", dist)
# Test unsuccessful trials are handled differently.
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()], state_fn=state_fn)
for i in range(32)
]
trial = frozen_trial_factory(32, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
partial_unsuccessful_suggestion = sampler.sample_independent(
study, trial, "param-a", dist)
assert partial_unsuccessful_suggestion != all_success_suggestion
def test_sample_independent_misc_arguments() -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 8)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
# Test misc. parameters.
sampler = TPESampler(n_ei_candidates=13, n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
sampler = TPESampler(gamma=lambda _: 5, n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
sampler = TPESampler(
weights=lambda i: np.asarray([i * 0.11 for i in range(7)]),
n_startup_trials=5,
seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
def test_sample_independent_prior() -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 8)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
sampler = TPESampler(consider_prior=False, n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
sampler = TPESampler(prior_weight=0.5, n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
def test_sample_independent_log_uniform_distributions() -> None:
"""Prepare sample from uniform distribution for cheking other distributions."""
study = optuna.create_study()
uni_dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=uni_dist) for i in range(1, 8)]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
uniform_suggestion = sampler.sample_independent(
study, trial, "param-a", uni_dist)
# Test sample from log-uniform is different from uniform.
log_dist = optuna.distributions.LogUniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=log_dist) for i in range(1, 8)]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
loguniform_suggestion = sampler.sample_independent(
study, trial, "param-a", log_dist)
assert 1.0 <= loguniform_suggestion < 100.0
assert uniform_suggestion != loguniform_suggestion
def test_sample_independent_handle_unsuccessful_states(
state: optuna.trial.TrialState) -> None:
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
# Prepare sampling result for later tests.
study = optuna.create_study()
for i in range(1, 30):
trial = frozen_trial_factory(i, dist=dist)
study._storage.create_new_trial(study._study_id, template_trial=trial)
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
all_success_suggestion = sampler.sample_independent(
study, trial, "param-a", dist)
# Test unsuccessful trials are handled differently.
state_fn = build_state_fn(state)
study = optuna.create_study()
for i in range(1, 30):
trial = frozen_trial_factory(i, dist=dist, state_fn=state_fn)
study._storage.create_new_trial(study._study_id, template_trial=trial)
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
partial_unsuccessful_suggestion = sampler.sample_independent(
study, trial, "param-a", dist)
assert partial_unsuccessful_suggestion != all_success_suggestion
def test_constrained_sample_independent_zero_startup() -> None:
"""Tests TPESampler with constrained option works when n_startup_trials=0."""
study = optuna.create_study()
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=0,
seed=2,
constraints_func=lambda _: (0, ))
sampler.sample_independent(study, trial, "param-a", dist)
def test_multi_objective_sample_independent_n_startup_trial() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
random.seed(128)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()]) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(n_startup_trials=16, seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials[:15]), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2, patch.object(
optuna.samplers.RandomSampler,
"sample_independent",
return_value=1.0,
) as sample_method:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
sampler.sample_independent(study, trial, "param-a", dist)
assert sample_method.call_count == 1
sampler = TPESampler(n_startup_trials=16, seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2, patch.object(
optuna.samplers.RandomSampler,
"sample_independent",
return_value=1.0,
) as sample_method:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
sampler.sample_independent(study, trial, "param-a", dist)
assert sample_method.call_count == 0
def test_multi_objective_sample_independent_prior() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
random.seed(128)
past_trials = [frozen_trial_factory(i, [random.random(), random.random()]) for i in range(16)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
sampler = TPESampler(consider_prior=False, seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a", dist) != suggestion
sampler = TPESampler(prior_weight=0.5, seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a", dist) != suggestion
def test_multi_objective_sample_independent_ignored_states() -> None:
"""Tests FAIL, RUNNING, and WAITING states are equally."""
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
suggestions = []
for state in [
optuna.trial.TrialState.FAIL,
optuna.trial.TrialState.RUNNING,
optuna.trial.TrialState.WAITING,
]:
random.seed(128)
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(i, [random.random(), random.random()], state_fn=state_fn)
for i in range(32)
]
trial = frozen_trial_factory(32, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
suggestions.append(sampler.sample_independent(study, trial, "param-a", dist))
assert len(set(suggestions)) == 1
def test_multi_objective_sample_int_uniform_distributions() -> None:
"""Test sampling from int distribution returns integer."""
study = optuna.create_study(directions=["minimize", "maximize"])
random.seed(128)
def int_value_fn(idx: int) -> float:
return random.randint(0, 100)
int_dist = optuna.distributions.IntUniformDistribution(1, 100)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()], dist=int_dist, value_fn=int_value_fn
)
for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
int_suggestion = sampler.sample_independent(study, trial, "param-a", int_dist)
assert 1 <= int_suggestion <= 100
assert isinstance(int_suggestion, int)
def test_multi_objective_sample_independent_categorical_distributions() -> None:
"""Test samples are drawn from the specified category."""
study = optuna.create_study(directions=["minimize", "maximize"])
random.seed(128)
categories = [i * 0.3 + 1.0 for i in range(330)]
def cat_value_fn(idx: int) -> float:
return categories[random.randint(0, len(categories) - 1)]
cat_dist = optuna.distributions.CategoricalDistribution(categories)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()], dist=cat_dist, value_fn=cat_value_fn
)
for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
categorical_suggestion = sampler.sample_independent(study, trial, "param-a", cat_dist)
assert categorical_suggestion in categories
def test_sample_independent_ignored_states() -> None:
"""Tests FAIL, RUNNING, and WAITING states are equally."""
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
suggestions = []
for state in [
optuna.trial.TrialState.FAIL,
optuna.trial.TrialState.RUNNING,
optuna.trial.TrialState.WAITING,
]:
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(i, dist=dist, state_fn=state_fn)
for i in range(1, 30)
]
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestions.append(
sampler.sample_independent(study, trial, "param-a", dist))
assert len(set(suggestions)) == 1
def test_sample_independent_pruned_state() -> None:
"""Tests PRUNED state is treated differently from both FAIL and COMPLETE."""
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
suggestions = []
for state in [
optuna.trial.TrialState.COMPLETE,
optuna.trial.TrialState.FAIL,
optuna.trial.TrialState.PRUNED,
]:
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(i, dist=dist, state_fn=state_fn)
for i in range(1, 30)
]
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestions.append(
sampler.sample_independent(study, trial, "param-a", dist))
assert len(set(suggestions)) == 3
def test_multi_objective_sample_independent_log_uniform_distributions() -> None:
"""Prepare sample from uniform distribution for cheking other distributions."""
study = optuna.create_study(directions=["minimize", "maximize"])
random.seed(128)
uni_dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, [random.random(), random.random()]) for i in range(16)]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
uniform_suggestion = sampler.sample_independent(study, trial, "param-a", uni_dist)
# Test sample from log-uniform is different from uniform.
log_dist = optuna.distributions.LogUniformDistribution(1.0, 100.0)
past_trials = [
frozen_trial_factory(i, [random.random(), random.random()], log_dist) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
loguniform_suggestion = sampler.sample_independent(study, trial, "param-a", log_dist)
assert 1.0 <= loguniform_suggestion < 100.0
assert uniform_suggestion != loguniform_suggestion
def test_sample_independent_uniform_distributions() -> None:
study = optuna.create_study()
# Prepare sample from uniform distribution for cheking other distributions.
uni_dist = optuna.distributions.FloatDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=uni_dist) for i in range(1, 8)]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage, "get_all_trials", return_value=past_trials):
uniform_suggestion = sampler.sample_independent(study, trial, "param-a", uni_dist)
assert 1.0 <= uniform_suggestion < 100.0
def test_sample_independent_n_startup_trial() -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 8)]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials[:4]):
with patch.object(optuna.samplers.random.RandomSampler,
"sample_independent",
return_value=1.0) as sample_method:
sampler.sample_independent(study, trial, "param-a", dist)
assert sample_method.call_count == 1
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials[:5]):
with patch.object(optuna.samplers.random.RandomSampler,
"sample_independent",
return_value=1.0) as sample_method:
sampler.sample_independent(study, trial, "param-a", dist)
assert sample_method.call_count == 0
def test_sample_independent_seed_fix() -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 8)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials):
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) == suggestion
sampler = TPESampler(n_startup_trials=5, seed=1)
with patch("optuna.Study.get_trials", return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
def test_sample_independent_disrete_uniform_distributions() -> None:
"""Test samples from discrete have expected intervals."""
study = optuna.create_study()
disc_dist = optuna.distributions.DiscreteUniformDistribution(1.0, 100.0, 0.1)
def value_fn(idx: int) -> float:
random.seed(idx)
return int(random.random() * 1000) * 0.1
past_trials = [frozen_trial_factory(i, dist=disc_dist, value_fn=value_fn) for i in range(1, 8)]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch("optuna.Study.get_trials", return_value=past_trials):
discrete_uniform_suggestion = sampler.sample_independent(
study, trial, "param-a", disc_dist
)
assert 1.0 <= discrete_uniform_suggestion <= 100.0
assert abs(int(discrete_uniform_suggestion * 10) - discrete_uniform_suggestion * 10) < 1e-3
def test_sample_independent_categorical_distributions() -> None:
"""Test samples are drawn from the specified category."""
study = optuna.create_study()
categories = [i * 0.3 + 1.0 for i in range(330)]
def cat_value_fn(idx: int) -> float:
random.seed(idx)
return categories[random.randint(0, len(categories) - 1)]
cat_dist = optuna.distributions.CategoricalDistribution(categories)
past_trials = [
frozen_trial_factory(i, dist=cat_dist, value_fn=cat_value_fn) for i in range(1, 8)
]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage, "get_all_trials", return_value=past_trials):
categorical_suggestion = sampler.sample_independent(study, trial, "param-a", cat_dist)
assert categorical_suggestion in categories
def test_sample_independent_int_loguniform_distributions() -> None:
"""Test sampling from int distribution returns integer."""
study = optuna.create_study()
def int_value_fn(idx: int) -> float:
random.seed(idx)
return random.randint(0, 100)
intlog_dist = optuna.distributions.IntLogUniformDistribution(1, 100)
past_trials = [
frozen_trial_factory(i, dist=intlog_dist, value_fn=int_value_fn) for i in range(1, 8)
]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage, "get_all_trials", return_value=past_trials):
intlog_suggestion = sampler.sample_independent(study, trial, "param-a", intlog_dist)
assert 1 <= intlog_suggestion <= 100
assert isinstance(intlog_suggestion, int)
def test_sample_independent_ignored_states() -> None:
"""Tests FAIL, RUNNING, and WAITING states are equally."""
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
suggestions = []
for state in [
optuna.trial.TrialState.FAIL,
optuna.trial.TrialState.RUNNING,
optuna.trial.TrialState.WAITING,
]:
study = optuna.create_study()
state_fn = build_state_fn(state)
for i in range(1, 30):
trial = frozen_trial_factory(i, dist=dist, state_fn=state_fn)
study._storage.create_new_trial(study._study_id, template_trial=trial)
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
suggestions.append(sampler.sample_independent(study, trial, "param-a", dist))
assert len(set(suggestions)) == 1
def test_sample_independent_pruned_state() -> None:
"""Tests PRUNED state is treated differently from both FAIL and COMPLETE."""
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
suggestions = []
for state in [
optuna.trial.TrialState.COMPLETE,
optuna.trial.TrialState.FAIL,
optuna.trial.TrialState.PRUNED,
]:
study = optuna.create_study()
state_fn = build_state_fn(state)
for i in range(1, 30):
trial = frozen_trial_factory(i, dist=dist, state_fn=state_fn)
study._storage.create_new_trial(study._study_id, template_trial=trial)
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=2)
suggestions.append(sampler.sample_independent(study, trial, "param-a", dist))
assert len(set(suggestions)) == 3
def test_multi_objective_sample_independent_disrete_uniform_distributions() -> None:
"""Test samples from discrete have expected intervals."""
study = optuna.create_study(directions=["minimize", "maximize"])
random.seed(128)
disc_dist = optuna.distributions.DiscreteUniformDistribution(1.0, 100.0, 0.1)
def value_fn(idx: int) -> float:
return int(random.random() * 1000) * 0.1
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()], dist=disc_dist, value_fn=value_fn
)
for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
discrete_uniform_suggestion = sampler.sample_independent(
study, trial, "param-a", disc_dist
)
assert 1.0 <= discrete_uniform_suggestion <= 100.0
assert abs(int(discrete_uniform_suggestion * 10) - discrete_uniform_suggestion * 10) < 1e-3
def test_multi_objective_sample_independent_float_distributions(
log: bool, step: Optional[float]) -> None:
# Prepare sample from float distribution for checking other distributions.
study = optuna.create_study(directions=["minimize", "maximize"])
random.seed(128)
float_dist = optuna.distributions.FloatDistribution(1.0,
100.0,
log=log,
step=step)
if float_dist.step:
value_fn: Optional[Callable[[int], float]] = (
lambda number: int(random.random() * 1000) * 0.1)
else:
value_fn = None
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()],
dist=float_dist,
value_fn=value_fn) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
float_suggestion = sampler.sample_independent(study, trial, "param-a",
float_dist)
assert 1.0 <= float_suggestion < 100.0
if float_dist.step == 0.1:
assert abs(int(float_suggestion * 10) - float_suggestion * 10) < 1e-3
# Test sample is different when `float_dist.log` is True or float_dist.step != 1.0.
random.seed(128)
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()]) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
if float_dist.log or float_dist.step == 0.1:
assert float_suggestion != suggestion
else:
assert float_suggestion == suggestion
def test_multi_objective_sample_independent_misc_arguments() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
random.seed(128)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()]) for i in range(32)
]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
# Test misc. parameters.
sampler = TPESampler(n_ei_candidates=13, seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
sampler = TPESampler(gamma=lambda _: 1, seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
sampler = TPESampler(weights=lambda n: np.zeros(n), seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
