def test_conditional_sample_independent(sampler_class: Callable[[], BaseSampler]) -> None:
# This test case reproduces the error reported in #2734.
# See https://github.com/optuna/optuna/pull/2734#issuecomment-857649769.
study = optuna.study.create_study(sampler=sampler_class())
categorical_distribution = CategoricalDistribution(choices=["x", "y"])
dependent_distribution = CategoricalDistribution(choices=["a", "b"])
study.add_trial(
optuna.create_trial(
params={"category": "x", "x": "a"},
distributions={"category": categorical_distribution, "x": dependent_distribution},
value=0.1,
)
)
study.add_trial(
optuna.create_trial(
params={"category": "y", "y": "b"},
distributions={"category": categorical_distribution, "y": dependent_distribution},
value=0.1,
)
)
_trial = _create_new_trial(study)
category = study.sampler.sample_independent(
study, _trial, "category", categorical_distribution
)
assert category in ["x", "y"]
value = study.sampler.sample_independent(study, _trial, category, dependent_distribution)
assert value in ["a", "b"]
def test_trials_dataframe_with_multi_objective_optimization_with_fail_and_pruned(
attrs: Tuple[str, ...], multi_index: bool) -> None:
study = create_study(directions=["minimize", "maximize"])
study.add_trial(create_trial(state=TrialState.FAIL))
study.add_trial(create_trial(state=TrialState.PRUNED))
df = study.trials_dataframe(attrs=attrs, multi_index=multi_index)
if multi_index:
for i in range(2):
assert df.get("values")[0][i] is None
assert df.get("values")[1][i] is None
else:
for i in range(2):
assert df.values_0[i] is None
assert df.values_1[i] is None
def after_trial(
self,
study,
trial,
state,
values,
):
if not self._warm_starting:
return self._sampler.after_trial(study, trial, state, values)
if len(self._source_trials) < self._warm_starting_trials:
assert state == optuna.trial.TrialState.PRUNED
self._source_trials.append(
optuna.create_trial(
params=trial.params,
distributions=trial.distributions,
values=values,
))
if len(self._source_trials) == self._warm_starting_trials:
self._sampler = optuna.samplers.CmaEsSampler(
seed=self._seed + 1, source_trials=self._source_trials or None)
self._warm_starting = False
else:
return self._sampler.after_trial(study, trial, state, values)
def __call__(self, study: "optuna.study.Study", trial: FrozenTrial) -> None:
system_attrs = {"failed_trial": trial.number}
# Update the new object with the values in the trial.system_attrs.
# By doing this, if this failed try is already a rety, the 'failed_trial' value
# will be the first failed trial number.
system_attrs.update(trial.system_attrs)
retries = sum(
("failed_trial", system_attrs["failed_trial"]) in t.system_attrs.items()
for t in study.trials
)
if self._max_retry is not None and retries + 1 > self._max_retry:
return
study.add_trial(
optuna.create_trial(
state=optuna.trial.TrialState.WAITING,
params=trial.params,
distributions=trial.distributions,
user_attrs=trial.user_attrs,
system_attrs=system_attrs,
intermediate_values=(
trial.intermediate_values if self._inherit_intermediate_values else None
),
)
)
def test_should_raise_exception() -> None:
dummy_source_trials = [create_trial(value=i, state=TrialState.COMPLETE) for i in range(10)]
with pytest.raises(ValueError):
optuna.samplers.CmaEsSampler(
x0={"x": 0.1, "y": 0.1},
source_trials=dummy_source_trials,
)
with pytest.raises(ValueError):
optuna.samplers.CmaEsSampler(
sigma0=0.1,
source_trials=dummy_source_trials,
)
with pytest.raises(ValueError):
optuna.samplers.CmaEsSampler(
use_separable_cma=True,
source_trials=dummy_source_trials,
)
with pytest.raises(ValueError):
optuna.samplers.CmaEsSampler(
restart_strategy="invalid-restart-strategy",
)
def test_different_distributions(self) -> None:
distributions = [
{
"x0": UniformDistribution(low=0, high=10),
"x1": UniformDistribution(low=0, high=10),
},
{
"x0": UniformDistribution(low=0, high=5),
"x1": UniformDistribution(low=0, high=10),
},
]
params = [
{
"x0": 0.5,
"x1": 0.5,
},
{
"x0": 0.5,
"x1": 0.5,
},
]
trials = [
create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p)
for d, p in zip(distributions, params)
]
search_space = _SearchSpace()
search_space.update(trials)
self.assertEqual(len(search_space.intersection), 1)
self.assertEqual(len(search_space.union), 3)
def test_plot_slice_log_scale():
# type: () -> None
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={
"x_linear": 1.0,
"y_log": 1e-3
},
distributions={
"x_linear": UniformDistribution(0.0, 3.0),
"y_log": LogUniformDistribution(1e-5, 1.0),
},
))
# Plot a parameter.
figure = plot_slice(study, params=["y_log"])
assert figure.layout["xaxis_type"] == "log"
figure = plot_slice(study, params=["x_linear"])
assert figure.layout["xaxis_type"] is None
# Plot multiple parameters.
figure = plot_slice(study)
assert figure.layout["xaxis_type"] is None
assert figure.layout["xaxis2_type"] == "log"
def test_restore_optimizer_keeps_backward_compatibility() -> None:
sampler = optuna.samplers.CmaEsSampler()
optimizer = CMA(np.zeros(2), sigma=1.3)
optimizer_str = pickle.dumps(optimizer).hex()
completed_trials = [
create_trial(state=TrialState.COMPLETE, value=0.1),
create_trial(
state=TrialState.COMPLETE,
value=0.1,
system_attrs={"cma:optimizer": optimizer_str, "cma:n_restarts": 1},
),
create_trial(state=TrialState.COMPLETE, value=0.1),
]
optimizer, n_restarts = sampler._restore_optimizer(completed_trials)
assert isinstance(optimizer, CMA)
assert n_restarts == 1
def test_add_trial(storage_mode: str) -> None:
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage)
assert len(study.trials) == 0
trial = create_trial(value=0.8)
study.add_trial(trial)
assert len(study.trials) == 1
assert study.trials[0].number == 0
assert study.best_value == 0.8
def test_restore_optimizer_from_substrings() -> None:
sampler = optuna.samplers.CmaEsSampler()
optimizer = CMA(np.zeros(10), sigma=1.3)
optimizer_str = pickle.dumps(optimizer).hex()
system_attrs: Dict[str, Any] = _split_optimizer_str(optimizer_str)
assert len(system_attrs) > 1
system_attrs["cma:n_restarts"] = 1
completed_trials = [
create_trial(state=TrialState.COMPLETE, value=0.1),
create_trial(
state=TrialState.COMPLETE,
value=0.1,
system_attrs=system_attrs,
),
create_trial(state=TrialState.COMPLETE, value=0.1),
]
optimizer, n_restarts = sampler._restore_optimizer(completed_trials)
assert isinstance(optimizer, CMA)
assert n_restarts == 1
def test_run_initialized(wandb: mock.MagicMock) -> None:
wandb.sdk.wandb_run.Run = mock.MagicMock
n_trials = 10
wandb_kwargs = {
"project": "optuna",
"group": "summary",
"job_type": "logging",
"mode": "offline",
"tags": ["test-tag"],
}
WeightsAndBiasesCallback(metric_name="mse",
wandb_kwargs=wandb_kwargs,
as_multirun=False)
wandb.init.assert_called_once_with(project="optuna",
group="summary",
job_type="logging",
mode="offline",
tags=["test-tag"])
wandbc = WeightsAndBiasesCallback(metric_name="mse",
wandb_kwargs=wandb_kwargs,
as_multirun=True)
wandb.run = None
study = optuna.create_study(direction="minimize")
_wrapped_func = wandbc.track_in_wandb()(lambda t: 1.0)
wandb.init.reset_mock()
trial = optuna.create_trial(value=1.0)
_wrapped_func(trial)
wandb.init.assert_called_once_with(project="optuna",
group="summary",
job_type="logging",
mode="offline",
tags=["test-tag"])
wandb.init.reset_mock()
study.optimize(_objective_func, n_trials=n_trials, callbacks=[wandbc])
wandb.init.assert_called_with(project="optuna",
group="summary",
job_type="logging",
mode="offline",
tags=["test-tag"])
assert wandb.init.call_count == n_trials
wandb.init().finish.assert_called()
assert wandb.init().finish.call_count == n_trials
def test_multi_objective_get_observation_pairs(
int_value: int,
categorical_value: optuna.distributions.CategoricalChoiceType,
objective_value: float,
multivariate: bool,
constant_liar: bool,
) -> None:
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
trial.suggest_int("x", int_value, int_value)
trial.suggest_categorical("y", [categorical_value])
return objective_value, objective_value
sampler = TPESampler(seed=0, multivariate=multivariate, constant_liar=constant_liar)
study = optuna.create_study(directions=["minimize", "maximize"], sampler=sampler)
study.optimize(objective, n_trials=2)
study.add_trial(
optuna.create_trial(
state=optuna.trial.TrialState.RUNNING,
params={"x": int_value, "y": categorical_value},
distributions={
"x": optuna.distributions.IntDistribution(int_value, int_value),
"y": optuna.distributions.CategoricalDistribution([categorical_value]),
},
)
)
assert _tpe.sampler._get_observation_pairs(study, ["x"], multivariate, constant_liar) == (
{"x": [int_value, int_value]},
[(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
None,
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], multivariate, constant_liar) == (
{"y": [0, 0]},
[(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
None,
)
assert _tpe.sampler._get_observation_pairs(study, ["x", "y"], True, constant_liar) == (
{"x": [int_value, int_value], "y": [0, 0]},
[(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
None,
)
assert _tpe.sampler._get_observation_pairs(study, ["z"], multivariate, constant_liar) == (
(
{"z": [None, None]},
[(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
None,
)
if not multivariate
else ({"z": []}, [], None)
)
def test_is_log_scale():
# type: () -> None
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={"param_linear": 1.0},
distributions={"param_linear": UniformDistribution(0.0, 3.0)},
))
study.add_trial(
create_trial(
value=2.0,
params={
"param_linear": 2.0,
"param_log": 1e-3
},
distributions={
"param_linear": UniformDistribution(0.0, 3.0),
"param_log": LogUniformDistribution(1e-5, 1.0),
},
))
assert _is_log_scale(study.trials, "param_log")
assert not _is_log_scale(study.trials, "param_linear")
def restart_from_checkpoint(study, trial):
# Enqueue trial with the same parameters as the stale trial to use saved information.
path = f"pytorch_checkpoint/{trial.number}/model.pt"
user_attrs = copy.deepcopy(trial.user_attrs)
if os.path.exists(path):
user_attrs["checkpoint_path"] = path
study.add_trial(
optuna.create_trial(
state=optuna.trial.TrialState.WAITING,
params=trial.params,
distributions=trial.distributions,
user_attrs=user_attrs,
system_attrs=trial.system_attrs,
))
def __call__(self, study: "optuna.study.Study", trial: FrozenTrial) -> None:
system_attrs = {"failed_trial": trial.number, "retry_history": [], **trial.system_attrs}
system_attrs["retry_history"].append(trial.number) # type: ignore
if self._max_retry is not None:
if self._max_retry < len(system_attrs["retry_history"]): # type: ignore
return
study.add_trial(
optuna.create_trial(
state=optuna.trial.TrialState.WAITING,
params=trial.params,
distributions=trial.distributions,
user_attrs=trial.user_attrs,
system_attrs=system_attrs,
intermediate_values=(
trial.intermediate_values if self._inherit_intermediate_values else None
),
)
)
def test_color_map(direction: str) -> None:
study = create_study(directions=[direction, direction])
for i in range(3):
study.add_trial(
create_trial(
values=[float(i), float(i)],
params={"param_a": 1.0, "param_b": 2.0},
distributions={
"param_a": FloatDistribution(0.0, 3.0),
"param_b": FloatDistribution(0.0, 3.0),
},
)
)
# Since `plot_pareto_front`'s colormap depends on only trial.number,
# `reversecale` is not in the plot.
marker = plot_pareto_front(study).data[0]["marker"]
assert COLOR_SCALE == [v[1] for v in marker["colorscale"]]
assert "reversecale" not in marker
def test_add_trials(storage_mode: str) -> None:
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage)
assert len(study.trials) == 0
study.add_trials([])
assert len(study.trials) == 0
trials = [create_trial(value=i) for i in range(3)]
study.add_trials(trials)
assert len(study.trials) == 3
for i, trial in enumerate(study.trials):
assert trial.number == i
assert trial.value == i
other_study = create_study(storage=storage)
other_study.add_trials(study.trials)
assert len(other_study.trials) == 3
for i, trial in enumerate(other_study.trials):
assert trial.number == i
assert trial.value == i
def test_plot_parallel_coordinate():
# type: () -> None
# Test with no trial.
study = create_study()
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
study = prepare_study_with_trials(with_c_d=False)
# Test with a trial.
figure = plot_parallel_coordinate(study)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][2]["values"] == (2.0, 0.0, 1.0)
# Test with a trial to select parameter.
figure = plot_parallel_coordinate(study, params=["param_a"])
assert len(figure.data[0]["dimensions"]) == 2
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
# Test with wrong params that do not exist in trials
with pytest.raises(ValueError, match="Parameter optuna does not exist in your study."):
plot_parallel_coordinate(study, params=["optuna", "optuna"])
# Ignore failed trials.
def fail_objective(_):
# type: (Trial) -> float
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError,))
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
# Test with categorical params that cannot be converted to numeral.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={"category_a": "preferred", "category_b": "net",},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
)
)
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={"category_a": "opt", "category_b": "una",},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
)
)
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 1)
assert figure.data[0]["dimensions"][1]["values"] == (0, 1)
assert figure.data[0]["dimensions"][1]["ticktext"] == (["preferred", 0], ["opt", 1])
assert figure.data[0]["dimensions"][2]["label"] == "category_b"
assert figure.data[0]["dimensions"][2]["range"] == (0, 1)
assert figure.data[0]["dimensions"][2]["values"] == (0, 1)
assert figure.data[0]["dimensions"][2]["ticktext"] == (["net", 0], ["una", 1])
# Load existed screenshots
distributions = {
"Body Type": CategoricalDistribution(choices=("Male", "Female")),
"Head": CategoricalDistribution(choices=list(range(20))),
"Brow Height": IntUniformDistribution(0, 12),
"Brow Depth": IntUniformDistribution(0, 12),
"Eyeline": IntUniformDistribution(0, 12),
"Eye Spacing": IntUniformDistribution(0, 12),
"Nose Width": IntUniformDistribution(0, 12),
"Nose Height": IntUniformDistribution(0, 12),
"Nose Bridge": IntUniformDistribution(0, 12),
"Mouth Height": IntUniformDistribution(0, 12),
"Cheeks": IntUniformDistribution(0, 12),
"Jawline": IntUniformDistribution(0, 12),
}
for filename, screenshot in tqdm(screenshot_folder.items(), ascii=True):
params = filename_to_params(filename)
encoding = get_face_encoding(screenshot)
distance = get_face_distance(encoding, target_encoding)
trial = optuna.create_trial(params=params,
distributions=distributions,
value=distance)
study.add_trial(trial)
study.optimize(objective, n_trials=10)
print()
print("Best settings:")
print_params(study.best_params)
