def test_fanova_importance_evaluator_with_infinite(inf_value: float) -> None:
# The test ensures that trials with infinite values are ignored to calculate importance scores.
n_trial = 10
seed = 13
# Importance scores are calculated without a trial with an inf value.
study = create_study(sampler=RandomSampler(seed=seed))
study.optimize(objective, n_trials=n_trial)
evaluator = FanovaImportanceEvaluator(seed=seed)
param_importance_without_inf = evaluator.evaluate(study)
# A trial with an inf value is added into the study manually.
study.add_trial(
create_trial(
value=inf_value,
params={"x1": 1.0, "x2": 1.0, "x3": 3.0},
distributions={
"x1": FloatDistribution(low=0.1, high=3),
"x2": FloatDistribution(low=0.1, high=3, log=True),
"x3": FloatDistribution(low=2, high=4, log=True),
},
)
)
# Importance scores are calculated with a trial with an inf value.
param_importance_with_inf = evaluator.evaluate(study)
# Obtained importance scores should be the same between with inf and without inf,
# because the last trial whose objective value is an inf is ignored.
assert param_importance_with_inf == param_importance_without_inf
def test_get_param_importances_invalid_params_type() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
with pytest.raises(TypeError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params={})
with pytest.raises(TypeError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=[0])
def test_fanova_importance_evaluator_with_target() -> None:
# Assumes that `seed` can be fixed to reproduce identical results.
study = create_study(sampler=RandomSampler(seed=0))
study.optimize(objective, n_trials=3)
evaluator = FanovaImportanceEvaluator(seed=0)
param_importance = evaluator.evaluate(study)
param_importance_with_target = evaluator.evaluate(
study,
target=lambda t: t.params["x1"] + t.params["x2"],
)
assert param_importance != param_importance_with_target
def test_get_param_importances_invalid_empty_study() -> None:
study = create_study()
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator())
def objective(trial: Trial) -> float:
raise optuna.exceptions.TrialPruned
study.optimize(objective, n_trials=3)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator())
def test_get_param_importances_with_params(
storage_init_func: Callable[[], storages.BaseStorage], params: List[str]
) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
x2 = trial.suggest_loguniform("x2", 0.1, 3)
x3 = trial.suggest_discrete_uniform("x3", 0, 3, 1)
if trial.number % 2 == 0:
x4 = trial.suggest_uniform("x4", 0.1, 3)
value = x1 ** 4 + x2 + x3
if trial.number % 2 == 0:
value += x4
return value
study = create_study(storage_init_func())
study.optimize(objective, n_trials=10)
param_importance = get_param_importances(
study, evaluator=FanovaImportanceEvaluator(), params=params
)
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == len(params)
assert all(param in param_importance for param in params)
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
if len(param_importance) > 0:
assert math.isclose(1.0, sum(i for i in param_importance.values()))
def test_get_param_importances(storage_init_func: Callable[[], storages.BaseStorage]) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
x2 = trial.suggest_loguniform("x2", 0.1, 3)
x3 = trial.suggest_discrete_uniform("x3", 0, 3, 1)
x4 = trial.suggest_int("x4", -3, 3)
x5 = trial.suggest_categorical("x5", [1.0, 1.1, 1.2])
if trial.number % 2 == 0:
# Conditional parameters are ignored unless `params` is specified and is not `None`.
x6 = trial.suggest_uniform("x6", 0.1, 3)
assert isinstance(x5, float)
value = x1 ** 4 + x2 + x3 - x4 ** 2 - x5
if trial.number % 2 == 0:
value += x6
return value
study = create_study(storage_init_func(), sampler=samplers.RandomSampler())
study.optimize(objective, n_trials=3)
param_importance = get_param_importances(study, evaluator=FanovaImportanceEvaluator())
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == 5
assert all(param_name in param_importance for param_name in ["x1", "x2", "x3", "x4", "x5"])
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
assert math.isclose(1.0, sum(i for i in param_importance.values()))
def test_fanova_importance_evaluator_seed() -> None:
study = create_study(sampler=RandomSampler(seed=0))
study.optimize(objective, n_trials=3)
evaluator = FanovaImportanceEvaluator(seed=2)
param_importance = evaluator.evaluate(study)
evaluator = FanovaImportanceEvaluator(seed=2)
param_importance_same_seed = evaluator.evaluate(study)
assert param_importance == param_importance_same_seed
evaluator = FanovaImportanceEvaluator(seed=3)
param_importance_different_seed = evaluator.evaluate(study)
assert param_importance != param_importance_different_seed
def test_get_param_importances_invalid_dynamic_search_space_params() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, trial.number + 0.1)
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x1"])
def test_get_param_importances_invalid_no_completed_trials_params() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
if trial.number % 2 == 0:
_ = trial.suggest_loguniform("x2", 0.1, 3)
raise optuna.exceptions.TrialPruned
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
# None of the trials with `x2` are completed.
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x2"])
# None of the trials with `x2` are completed. Adding "x1" should not matter.
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x1", "x2"])
# None of the trials contain `x3`.
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x3"])
def test_fanova_importance_evaluator_max_depth() -> None:
# Assumes that `seed` can be fixed to reproduce identical results.
study = create_study(sampler=RandomSampler(seed=0))
study.optimize(objective, n_trials=3)
evaluator = FanovaImportanceEvaluator(max_depth=1, seed=0)
param_importance = evaluator.evaluate(study)
evaluator = FanovaImportanceEvaluator(max_depth=2, seed=0)
param_importance_different_max_depth = evaluator.evaluate(study)
assert param_importance != param_importance_different_max_depth
def test_fanova_importance_evaluator_n_trees() -> None:
# Assumes that `seed` can be fixed to reproduce identical results.
study = create_study()
study.optimize(objective, n_trials=3)
evaluator = FanovaImportanceEvaluator(n_trees=10, seed=0)
param_importance = evaluator.evaluate(study)
evaluator = FanovaImportanceEvaluator(n_trees=20, seed=0)
param_importance_different_n_trees = evaluator.evaluate(study)
assert param_importance != param_importance_different_n_trees
study.optimize(_objective, n_trials=2)
ax = plot_param_importances(study)
# Test if label for `y` param has been switched to `<0.01`.
labels = ax.figure.findobj(lambda obj: "<0.01" in str(obj))
assert len(labels) == 1
plt.savefig(BytesIO())
@pytest.mark.parametrize("inf_value", [float("inf"), -float("inf")])
@pytest.mark.parametrize(
"evaluator",
[
MeanDecreaseImpurityImportanceEvaluator(seed=10),
FanovaImportanceEvaluator(seed=10)
],
)
@pytest.mark.parametrize("n_trial", [0, 10])
def test_trial_with_infinite_value_ignored(inf_value: float,
evaluator: BaseImportanceEvaluator,
n_trial: int) -> None:
def _objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 2, 4, log=True)
return x1 + x2 * x3
seed = 13
study = create_study(sampler=RandomSampler(seed=seed))
study = create_study()
for x in range(1, 3):
study.enqueue_trial({"x": x, "y": 0})
study.optimize(_objective, n_trials=2)
figure = plot_param_importances(study)
# Test if label for `y` param has been switched to `<0.01`.
labels = figure.data[0].text
assert labels == ("<0.01", "1.00")
@pytest.mark.parametrize("inf_value", [float("inf"), -float("inf")])
@pytest.mark.parametrize(
"evaluator",
[MeanDecreaseImpurityImportanceEvaluator(seed=10), FanovaImportanceEvaluator(seed=10)],
)
@pytest.mark.parametrize("n_trial", [0, 10])
def test_trial_with_infinite_value_ignored(
inf_value: int, evaluator: BaseImportanceEvaluator, n_trial: int
) -> None:
def _objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 2, 4, log=True)
return x1 + x2 * x3
seed = 13
study = create_study(sampler=RandomSampler(seed=seed))
study.optimize(_objective, n_trials=n_trial)
