def test_sample_independent_pruned_state() -> None:
"""Tests PRUNED state is treated differently from both FAIL and COMPLETE."""
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
suggestions = []
for state in [
optuna.trial.TrialState.COMPLETE,
optuna.trial.TrialState.FAIL,
optuna.trial.TrialState.PRUNED,
]:
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(i, dist=dist, state_fn=state_fn)
for i in range(1, 30)
]
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestions.append(
sampler.sample_independent(study, trial, "param-a", dist))
assert len(set(suggestions)) == 3
def test_hyperopt_parameters(use_hyperband: bool) -> None:
sampler = TPESampler(**TPESampler.hyperopt_parameters())
study = optuna.create_study(
sampler=sampler,
pruner=optuna.pruners.HyperbandPruner() if use_hyperband else None)
study.optimize(lambda t: t.suggest_uniform("x", 10, 20), n_trials=50)
def test_sample_independent_ignored_states() -> None:
"""Tests FAIL, RUNNING, and WAITING states are equally."""
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
suggestions = []
for state in [
optuna.trial.TrialState.FAIL,
optuna.trial.TrialState.RUNNING,
optuna.trial.TrialState.WAITING,
]:
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(i, dist=dist, state_fn=state_fn)
for i in range(1, 30)
]
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestions.append(
sampler.sample_independent(study, trial, "param-a", dist))
assert len(set(suggestions)) == 1
def test_multi_objective_sample_independent_handle_unsuccessful_states(
state: optuna.trial.TrialState, ) -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
random.seed(128)
# Prepare sampling result for later tests.
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()]) for i in range(32)
]
trial = frozen_trial_factory(32, [0, 0])
sampler = TPESampler(seed=0)
all_success_suggestion = suggest(sampler, study, trial, dist, past_trials)
# Test unsuccessful trials are handled differently.
state_fn = build_state_fn(state)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()], state_fn=state_fn)
for i in range(32)
]
trial = frozen_trial_factory(32, [0, 0])
sampler = TPESampler(seed=0)
partial_unsuccessful_suggestion = suggest(sampler, study, trial, dist,
past_trials)
assert partial_unsuccessful_suggestion != all_success_suggestion
def test_sample_relative_empty_input(multivariate: bool) -> None:
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(multivariate=multivariate)
# A frozen-trial is not supposed to be accessed.
study = optuna.create_study()
frozen_trial = Mock(spec=[])
assert sampler.sample_relative(study, frozen_trial, {}) == {}
def test_constrained_sample_independent_zero_startup() -> None:
"""Tests TPESampler with constrained option works when n_startup_trials=0."""
study = optuna.create_study()
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
trial = frozen_trial_factory(30)
sampler = TPESampler(n_startup_trials=0,
seed=2,
constraints_func=lambda _: (0, ))
sampler.sample_independent(study, trial, "param-a", dist)
def test_reseed_rng() -> None:
sampler = TPESampler()
original_seed = sampler._rng.seed
with patch.object(sampler._random_sampler,
"reseed_rng",
wraps=sampler._random_sampler.reseed_rng) as mock_object:
sampler.reseed_rng()
assert mock_object.call_count == 1
assert original_seed != sampler._rng.seed
def test_sample_independent_prior() -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 8)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
sampler = TPESampler(consider_prior=False, n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
sampler = TPESampler(prior_weight=0.5, n_startup_trials=5, seed=0)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_independent(study, trial, "param-a",
dist) != suggestion
def test_sample_relative_seed_fix() -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 8)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(8)
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(n_startup_trials=5, seed=0, multivariate=True)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
suggestion = sampler.sample_relative(study, trial, {"param-a": dist})
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(n_startup_trials=5, seed=0, multivariate=True)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_relative(study, trial,
{"param-a": dist}) == suggestion
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(n_startup_trials=5, seed=1, multivariate=True)
with patch.object(study._storage,
"get_all_trials",
return_value=past_trials):
assert sampler.sample_relative(study, trial,
{"param-a": dist}) != suggestion
def test_sample_independent_uniform_distributions() -> None:
study = optuna.create_study()
# Prepare sample from uniform distribution for cheking other distributions.
uni_dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=uni_dist) for i in range(1, 8)]
trial = frozen_trial_factory(8)
sampler = TPESampler(n_startup_trials=5, seed=0)
with patch.object(study._storage, "get_all_trials", return_value=past_trials):
uniform_suggestion = sampler.sample_independent(study, trial, "param-a", uni_dist)
assert 1.0 <= uniform_suggestion < 100.0
def test_sample_relative_uniform_distributions() -> None:
study = optuna.create_study()
# Prepare sample from uniform distribution for cheking other distributions.
uni_dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=uni_dist) for i in range(1, 8)]
trial = frozen_trial_factory(8)
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(n_startup_trials=5, seed=0, multivariate=True)
with patch.object(study._storage, "get_all_trials", return_value=past_trials):
uniform_suggestion = sampler.sample_relative(study, trial, {"param-a": uni_dist})
assert 1.0 <= uniform_suggestion["param-a"] < 100.0
def test_sample_relative_n_startup_trial() -> None:
study = optuna.create_study()
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
past_trials = [frozen_trial_factory(i, dist=dist) for i in range(1, 8)]
trial = frozen_trial_factory(8)
# sample_relative returns {} for only 4 observations.
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(n_startup_trials=5, seed=0, multivariate=True)
with patch.object(study._storage, "get_all_trials", return_value=past_trials[:4]):
assert sampler.sample_relative(study, trial, {"param-a": dist}) == {}
# sample_relative returns some value for only 7 observations.
with patch.object(study._storage, "get_all_trials", return_value=past_trials):
assert "param-a" in sampler.sample_relative(study, trial, {"param-a": dist}).keys()
def test_multi_objective_sample_independent_categorical_distributions(
) -> None:
"""Test samples are drawn from the specified category."""
study = optuna.create_study(directions=["minimize", "maximize"])
random.seed(128)
categories = [i * 0.3 + 1.0 for i in range(330)]
def cat_value_fn(idx: int) -> float:
return categories[random.randint(0, len(categories) - 1)]
cat_dist = optuna.distributions.CategoricalDistribution(categories)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()],
dist=cat_dist,
value_fn=cat_value_fn) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
categorical_suggestion = suggest(sampler, study, trial, cat_dist,
past_trials)
assert categorical_suggestion in categories
def test_multi_objective_sample_independent_seed_fix() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
random.seed(128)
past_trials = [frozen_trial_factory(i, [random.random(), random.random()]) for i in range(16)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
suggestion = suggest(sampler, study, trial, dist, past_trials)
sampler = TPESampler(seed=0)
assert suggest(sampler, study, trial, dist, past_trials) == suggestion
sampler = TPESampler(seed=1)
assert suggest(sampler, study, trial, dist, past_trials) != suggestion
def test_multi_objective_get_observation_pairs_constrained(constraint_value: int) -> None:
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
trial.suggest_int("x", 5, 5)
trial.set_user_attr("constraint", (constraint_value, -1))
return 5.0, 5.0
sampler = TPESampler(constraints_func=lambda trial: trial.user_attrs["constraint"], seed=0)
study = optuna.create_study(directions=["minimize", "maximize"], sampler=sampler)
study.optimize(objective, n_trials=5)
violations = [max(0, constraint_value) for _ in range(5)]
assert _tpe.sampler._get_observation_pairs(study, ["x"], False, constraints_enabled=True) == (
{"x": [5.0, 5.0, 5.0, 5.0, 5.0]},
[(-float("inf"), [5.0, -5.0]) for _ in range(5)],
violations,
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], False, constraints_enabled=True) == (
{"y": [None, None, None, None, None]},
[(-float("inf"), [5.0, -5.0]) for _ in range(5)],
violations,
)
assert _tpe.sampler._get_observation_pairs(study, ["x"], True, constraints_enabled=True) == (
{"x": [5.0, 5.0, 5.0, 5.0, 5.0]},
[(-float("inf"), [5.0, -5.0]) for _ in range(5)],
violations,
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], True, constraints_enabled=True) == (
{"y": []},
[],
[],
)
def test_constant_liar_observation_pairs(direction: str, multivariate: bool) -> None:
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(constant_liar=True)
study = optuna.create_study(sampler=sampler, direction=direction)
trial = study.ask()
trial.suggest_int("x", 2, 2)
assert (
len(study.trials) == 1 and study.trials[0].state == optuna.trial.TrialState.RUNNING
), "Precondition"
# The value of the constant liar should be penalizing, i.e. `float("inf")` during minimization
# and `-float("inf")` during maximization.
expected_values = [(-float("inf"), [float("inf") * (-1 if direction == "maximize" else 1)])]
assert _tpe.sampler._get_observation_pairs(
study, ["x"], multivariate, constant_liar=False
) == (
{"x": []},
[],
)
assert _tpe.sampler._get_observation_pairs(study, ["x"], multivariate, constant_liar=True) == (
{"x": [2]},
expected_values,
)
def run_study(n_trials, storage, args_path, tune_path, n_same_runs):
study = optuna.create_study(
storage=storage, study_name="td3", load_if_exists=True,
direction="maximize", sampler=TPESampler())
study.optimize(lambda trial: objective(
trial, args_path, tune_path, n_same_runs), n_trials=n_trials)
return study
def tune(self, n_trials: int = 30) -> None:
"""Hyperparameter Tuning by TPE."""
path = Path(f'../logs/{self.data}')
path.mkdir(exist_ok=True, parents=True)
path_model = Path(f'../logs/{self.data}/{self.model_name}')
path_model.mkdir(exist_ok=True, parents=True)
# Tune hyperparameters by Optuna
objective = Objective(data=self.data, model_name=self.model_name)
study = optuna.create_study(sampler=TPESampler(seed=rand_seed_val_tpe))
study.optimize(objective, n_trials=n_trials)
# Save tuning results as a YAML file
study.trials_dataframe().to_csv(str(path_model /
f'tuning_results.csv'))
if Path('../hyper_params.yaml').exists():
pass
else:
yaml.dump(dict(yahoo=dict(), coat=dict()),
open('../hyper_params.yaml', 'w'),
default_flow_style=False)
time.sleep(np.random.rand())
hyper_params_dict = yaml.safe_load(open('../hyper_params.yaml', 'r'))
hyper_params_dict[self.data][self.model_name] = study.best_params
yaml.dump(hyper_params_dict,
open('../hyper_params.yaml', 'w'),
default_flow_style=False)
def test_multi_objective_get_observation_pairs() -> None:
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
trial.suggest_int("x", 5, 5)
return 5.0, 5.0
sampler = TPESampler(seed=0)
study = optuna.create_study(directions=["minimize", "maximize"],
sampler=sampler)
study.optimize(objective, n_trials=5)
assert _tpe.sampler._get_observation_pairs(study, ["x"], False) == (
{
"x": [5.0, 5.0, 5.0, 5.0, 5.0]
},
[(-float("inf"), [5.0, -5.0]) for _ in range(5)],
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], False) == (
{
"y": [None, None, None, None, None]
},
[(-float("inf"), [5.0, -5.0]) for _ in range(5)],
)
assert _tpe.sampler._get_observation_pairs(study, ["x"], True) == (
{
"x": [5.0, 5.0, 5.0, 5.0, 5.0]
},
[(-float("inf"), [5.0, -5.0]) for _ in range(5)],
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], True) == ({
"y": []
}, [])
def search_one(
pipe: Any,
X: InteractionMatrix,
evaluator: Evaluator,
optimizer_names: List[str],
suggest_overwrites: Dict[str, List[Suggestion]],
db_url: str,
study_name: str,
random_seed: int,
logger: Logger,
) -> None:
study = optuna.load_study(
storage=db_url,
study_name=study_name,
sampler=TPESampler(seed=random_seed),
)
def _obj(trial: optuna.Trial) -> float:
pipe.send(trial.number)
optimizer_name = trial.suggest_categorical("optimizer_name", optimizer_names)
assert isinstance(optimizer_name, str)
optimizer = get_optimizer_class(optimizer_name)(
X,
evaluator,
suggest_overwrite=suggest_overwrites[optimizer_name],
logger=logger,
)
return optimizer.objective_function(optimizer_name + ".")(trial)
study.optimize(_obj, n_trials=1)
def _make_sampler(self):
if self.sampler_method == 'random':
sampler = RandomSampler(seed=self.seed)
elif self.sampler_method == 'tpe':
sampler = TPESampler(n_startup_trials=5, seed=self.seed)
else:
raise ValueError('Unknown sampler: {}'.format(self.sampler_method))
return sampler
def test_call_after_trial_of_random_sampler() -> None:
sampler = TPESampler()
study = optuna.create_study(sampler=sampler)
with patch.object(
sampler._random_sampler, "after_trial", wraps=sampler._random_sampler.after_trial
) as mock_object:
study.optimize(lambda _: 1.0, n_trials=1)
assert mock_object.call_count == 1
def run_tpe(k: int, sequence_dict: Dict[int, List[int]], hash_dict: Dict[int, int]) -> None:
hash_dict[k] = hash("nondeterministic hash")
sampler = TPESampler(n_startup_trials=1, seed=2, multivariate=True, group=True)
study = create_study(sampler=sampler)
study.optimize(
lambda t: np.sum([t.suggest_int(f"x{i}", 0, 10) for i in range(10)]), n_trials=2
)
sequence_dict[k] = list(study.trials[-1].params.values())
def test_multi_objective_sample_independent_misc_arguments() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
random.seed(128)
past_trials = [frozen_trial_factory(i, [random.random(), random.random()]) for i in range(32)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
suggestion = suggest(sampler, study, trial, dist, past_trials)
# Test misc. parameters.
sampler = TPESampler(n_ei_candidates=13, seed=0)
assert suggest(sampler, study, trial, dist, past_trials) != suggestion
sampler = TPESampler(gamma=lambda _: 1, seed=0)
assert suggest(sampler, study, trial, dist, past_trials) != suggestion
def test_group() -> None:
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(multivariate=True, group=True)
study = optuna.create_study(sampler=sampler)
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_int("x", 0, 10), n_trials=2)
assert mock.call_count == 1
assert study.trials[-1].distributions == {"x": distributions.IntDistribution(low=0, high=10)}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10) + t.suggest_float("z", -3, 3), n_trials=1
)
assert mock.call_count == 1
assert study.trials[-1].distributions == {
"y": distributions.IntDistribution(low=0, high=10),
"z": distributions.FloatDistribution(low=-3, high=3),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10)
+ t.suggest_float("z", -3, 3)
+ t.suggest_float("u", 1e-2, 1e2, log=True)
+ bool(t.suggest_categorical("v", ["A", "B", "C"])),
n_trials=1,
)
assert mock.call_count == 2
assert study.trials[-1].distributions == {
"u": distributions.FloatDistribution(low=1e-2, high=1e2, log=True),
"v": distributions.CategoricalDistribution(choices=["A", "B", "C"]),
"y": distributions.IntDistribution(low=0, high=10),
"z": distributions.FloatDistribution(low=-3, high=3),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_float("u", 1e-2, 1e2, log=True), n_trials=1)
assert mock.call_count == 3
assert study.trials[-1].distributions == {
"u": distributions.FloatDistribution(low=1e-2, high=1e2, log=True)
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10) + t.suggest_int("w", 2, 8, log=True), n_trials=1
)
assert mock.call_count == 4
assert study.trials[-1].distributions == {
"y": distributions.IntDistribution(low=0, high=10),
"w": distributions.IntDistribution(low=2, high=8, log=True),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_int("x", 0, 10), n_trials=1)
assert mock.call_count == 6
assert study.trials[-1].distributions == {"x": distributions.IntDistribution(low=0, high=10)}
def test_multi_objective_sample_independent_float_distributions(
log: bool, step: Optional[float]) -> None:
# Prepare sample from float distribution for checking other distributions.
study = optuna.create_study(directions=["minimize", "maximize"])
random.seed(128)
float_dist = optuna.distributions.FloatDistribution(1.0,
100.0,
log=log,
step=step)
if float_dist.step:
value_fn: Optional[Callable[[int], float]] = (
lambda number: int(random.random() * 1000) * 0.1)
else:
value_fn = None
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()],
dist=float_dist,
value_fn=value_fn) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
float_suggestion = suggest(sampler, study, trial, float_dist, past_trials)
assert 1.0 <= float_suggestion < 100.0
if float_dist.step == 0.1:
assert abs(int(float_suggestion * 10) - float_suggestion * 10) < 1e-3
# Test sample is different when `float_dist.log` is True or float_dist.step != 1.0.
random.seed(128)
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()]) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
suggestion = suggest(sampler, study, trial, dist, past_trials)
if float_dist.log or float_dist.step == 0.1:
assert float_suggestion != suggestion
else:
assert float_suggestion == suggestion
def test_multi_objective_sample_independent_n_startup_trial() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.FloatDistribution(1.0, 100.0)
random.seed(128)
past_trials = [
frozen_trial_factory(
i, [random.random(), random.random()]) for i in range(16)
]
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(n_startup_trials=16, seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials[:15]), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2, patch.object(
optuna.samplers.RandomSampler,
"sample_independent",
return_value=1.0,
) as sample_method:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
sampler.sample_independent(study, trial, "param-a", dist)
assert sample_method.call_count == 1
sampler = TPESampler(n_startup_trials=16, seed=0)
attrs = MockSystemAttr()
with patch.object(
study._storage, "get_all_trials",
return_value=past_trials), patch.object(
study._storage,
"set_trial_system_attr",
side_effect=attrs.set_trial_system_attr), patch.object(
study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs",
new_callable=PropertyMock) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2, patch.object(
optuna.samplers.RandomSampler,
"sample_independent",
return_value=1.0,
) as sample_method:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
sampler.sample_independent(study, trial, "param-a", dist)
assert sample_method.call_count == 0
def __init__(self, frameworks,
model_types,
pre_proc=None,
proc_dict=None,
post_proc=None,
scoring=None,
cv=3,
agg_func=np.mean,
tol=1e-7,
max_iter=50,
max_fails=0,
time_limit=None,
study_name=None,
save_cv_preds=False,
pruner=LinearExtrapolationPruner(n_steps_back=2, n_steps_forward=15, percentage_from_best=90),
sampler=TPESampler(**TPESampler.hyperopt_parameters()),
storage=None,
refit=True,
n_jobs=1,
verbose=1,
random_state=None):
self.frameworks = frameworks if isinstance(frameworks, list) else [frameworks]
self.model_types = model_types if isinstance(model_types, list) else [model_types]
self.pre_proc = pre_proc,
self.proc_dict = proc_dict,
self.post_proc = post_proc,
self.scoring = scoring
self.cv = cv
self.agg_func = agg_func
self.refit_ = refit
self.tol = tol
self.max_iter = max_iter
self.time_limit = time_limit
self.max_fails = max_fails
self.study_name = study_name
self.save_cv_preds = save_cv_preds
self.pruner = pruner
self.sampler = sampler
# TODO: add CMA Sampler
self.storage = storage
self.n_jobs = n_jobs
self.verbose = verbose
self.random_state = random_state
self.best_pipeline_ = None
def test_multi_objective_sample_independent_prior() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
random.seed(128)
past_trials = [frozen_trial_factory(i, [random.random(), random.random()]) for i in range(16)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(16, [0, 0])
sampler = TPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
sampler = TPESampler(consider_prior=False, seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a", dist) != suggestion
sampler = TPESampler(prior_weight=0.5, seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a", dist) != suggestion
def create_sampler(sampler_mode: str) -> BaseSampler:
if sampler_mode == "random":
return RandomSampler()
elif sampler_mode == "tpe":
return TPESampler()
elif sampler_mode == "cmaes":
return CmaEsSampler()
else:
assert False
