def test_get_param_importance_target_is_none_and_study_is_multi_obj(
storage_mode: str,
evaluator_init_func: Callable[[], BaseImportanceEvaluator],
) -> None:
def objective(trial: Trial) -> Tuple[float, float]:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 0, 3, step=1)
x4 = trial.suggest_int("x4", -3, 3)
x5 = trial.suggest_int("x5", 1, 5, log=True)
x6 = trial.suggest_categorical("x6", [1.0, 1.1, 1.2])
if trial.number % 2 == 0:
# Conditional parameters are ignored unless `params` is specified and is not `None`.
x7 = trial.suggest_float("x7", 0.1, 3)
assert isinstance(x6, float)
value = x1**4 + x2 + x3 - x4**2 - x5 + x6
if trial.number % 2 == 0:
value += x7
return value, 0.0
with StorageSupplier(storage_mode) as storage:
study = create_study(directions=["minimize", "minimize"],
storage=storage)
study.optimize(objective, n_trials=3)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=evaluator_init_func())
def test_get_param_importances_unnormalized_experimental() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
return x1**2
study = create_study()
study.optimize(objective, n_trials=4)
with pytest.warns(ExperimentalWarning):
get_param_importances(study, normalize=False)
def test_get_param_importances_invalid_dynamic_search_space_params() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, trial.number + 0.1)
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x1"])
def test_get_param_importances_invalid_single_trial() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=1)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator())
def test_get_param_importances_invalid_evaluator_type() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
with pytest.raises(TypeError):
get_param_importances(study, evaluator={}) # type: ignore
def test_get_param_importances_invalid_single_trial(
evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
return x1**2
study = create_study()
study.optimize(objective, n_trials=1)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=evaluator_init_func())
def test_get_param_importances_invalid_empty_study(
evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None:
study = create_study()
with pytest.raises(ValueError):
get_param_importances(study, evaluator=evaluator_init_func())
study.optimize(pruned_objective, n_trials=3)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=evaluator_init_func())
def test_get_param_importances_invalid_dynamic_search_space_params(
evaluator_init_func: Callable[[], BaseImportanceEvaluator]
) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, trial.number + 0.1)
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=evaluator_init_func(), params=["x1"])
def test_get_param_importances_invalid_empty_study() -> None:
study = create_study()
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator())
def objective(trial: Trial) -> float:
raise optuna.exceptions.TrialPruned
study.optimize(objective, n_trials=3)
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator())
def test_get_param_importances_invalid_params_type(
evaluator_init_func: Callable[[], BaseImportanceEvaluator]
) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
with pytest.raises(TypeError):
get_param_importances(study, evaluator=evaluator_init_func(), params={}) # type: ignore
with pytest.raises(TypeError):
get_param_importances(study, evaluator=evaluator_init_func(), params=[0]) # type: ignore
def test_get_param_importances_with_params(
storage_init_func: Callable[[], storages.BaseStorage],
params: List[str],
evaluator_init_func: Callable[[], BaseImportanceEvaluator],
) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 0, 3, step=1)
if trial.number % 2 == 0:
x4 = trial.suggest_float("x4", 0.1, 3)
value = x1 ** 4 + x2 + x3
if trial.number % 2 == 0:
value += x4
return value
study = create_study(storage_init_func())
study.optimize(objective, n_trials=10)
param_importance = get_param_importances(study, evaluator=evaluator_init_func(), params=params)
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == len(params)
assert all(param in param_importance for param in params)
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
if len(param_importance) > 0:
assert math.isclose(1.0, sum(i for i in param_importance.values()), abs_tol=1e-5)
def test_get_param_importances(storage_init_func: Callable[[], storages.BaseStorage]) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
x2 = trial.suggest_loguniform("x2", 0.1, 3)
x3 = trial.suggest_discrete_uniform("x3", 0, 3, 1)
x4 = trial.suggest_int("x4", -3, 3)
x5 = trial.suggest_categorical("x5", [1.0, 1.1, 1.2])
if trial.number % 2 == 0:
# Conditional parameters are ignored unless `params` is specified and is not `None`.
x6 = trial.suggest_uniform("x6", 0.1, 3)
assert isinstance(x5, float)
value = x1 ** 4 + x2 + x3 - x4 ** 2 - x5
if trial.number % 2 == 0:
value += x6
return value
study = create_study(storage_init_func(), sampler=samplers.RandomSampler())
study.optimize(objective, n_trials=3)
param_importance = get_param_importances(study, evaluator=FanovaImportanceEvaluator())
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == 5
assert all(param_name in param_importance for param_name in ["x1", "x2", "x3", "x4", "x5"])
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
assert math.isclose(1.0, sum(i for i in param_importance.values()))
def test_get_param_importances_with_params(
storage_mode: str,
params: List[str],
evaluator_init_func: Callable[[], BaseImportanceEvaluator],
) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 0, 3, step=1)
if trial.number % 2 == 0:
x4 = trial.suggest_float("x4", 0.1, 3)
value = x1**4 + x2 + x3
if trial.number % 2 == 0:
value += x4
return value
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage)
study.optimize(objective, n_trials=10)
param_importance = get_param_importances(
study, evaluator=evaluator_init_func(), params=params)
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == len(params)
assert all(param in param_importance for param in params)
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
# Sanity check for param importances
assert all(0 <= x < float("inf") for x in param_importance.values())
def test_get_param_importances_with_target(
storage_init_func: Callable[[], storages.BaseStorage],
evaluator_init_func: Callable[[], BaseImportanceEvaluator],
) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 0, 3, step=1)
if trial.number % 2 == 0:
x4 = trial.suggest_float("x4", 0.1, 3)
value = x1 ** 4 + x2 + x3
if trial.number % 2 == 0:
value += x4
return value
study = create_study(storage_init_func())
study.optimize(objective, n_trials=3)
param_importance = get_param_importances(
study,
evaluator=evaluator_init_func(),
target=lambda t: t.params["x1"] + t.params["x2"],
)
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == 3
assert all(param_name in param_importance for param_name in ["x1", "x2", "x3"])
prev_importance = float("inf")
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
assert importance <= prev_importance
prev_importance = importance
assert math.isclose(1.0, sum(param_importance.values()), abs_tol=1e-5)
def plot_param_importances(
study: Study,
evaluator = None,
params: Optional[List[str]] = None,
*,
target: Optional[Callable[[FrozenTrial], float]] = None,
target_name: str = "Objective Value",
):
_imports.check()
_check_plot_args(study, target, target_name)
layout = go.Layout(
title="Hyperparameter Importances",
xaxis={"title": f"Importance for {target_name}"},
yaxis={"title": "Hyperparameter"},
showlegend=False,
)
# Importances cannot be evaluated without completed trials.
# Return an empty figure for consistency with other visualization functions.
trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE]
if len(trials) == 0:
logger.warning("Study instance does not contain completed trials.")
return go.Figure(data=[], layout=layout)
if evaluator is None:
evaluator = ImportanceEvaluator()
try:
importances, importance_paras = get_param_importances(
study, evaluator=evaluator, params=params, target=target
)
except RuntimeError: # sometimes it is returning error e.g. when number of trials are < 4
return None, None, None
importances = OrderedDict(reversed(list(importances.items())))
importance_values = list(importances.values())
param_names = list(importances.keys())
fig = go.Figure(
data=[
go.Bar(
x=importance_values,
y=param_names,
text=importance_values,
texttemplate="%{text:.2f}",
textposition="outside",
cliponaxis=False, # Ensure text is not clipped.
hovertemplate=[
_make_hovertext(param_name, importance, study)
for param_name, importance in importances.items()
],
marker_color=[_get_color(param_name, study) for param_name in param_names],
orientation="h",
)
],
layout=layout,
)
return importances, importance_paras, fig
def test_get_param_importances_invalid_no_completed_trials_params(
evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
if trial.number % 2 == 0:
_ = trial.suggest_float("x2", 0.1, 3, log=True)
raise optuna.TrialPruned
return x1**2
study = create_study()
study.optimize(objective, n_trials=3)
# None of the trials with `x2` are completed.
with pytest.raises(ValueError):
get_param_importances(study,
evaluator=evaluator_init_func(),
params=["x2"])
# None of the trials with `x2` are completed. Adding "x1" should not matter.
with pytest.raises(ValueError):
get_param_importances(study,
evaluator=evaluator_init_func(),
params=["x1", "x2"])
# None of the trials contain `x3`.
with pytest.raises(ValueError):
get_param_importances(study,
evaluator=evaluator_init_func(),
params=["x3"])
def save_metrics(self):
trials_importance: OrderedDict = get_param_importances(self.study)
with open(
"./data_processing/data/results/" + self.study_name +
"_importance.csv", "w") as csv_file:
w = csv.writer(csv_file)
for key, val in trials_importance.items():
w.writerow([key, val])
trails_df = self.study.trials_dataframe()
trails_df.to_csv("./data_processing/data/results/" + self.study_name +
"_history.csv",
mode='w',
index=False)
def test_get_param_importances_empty_search_space(
evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None:
def objective(trial: Trial) -> float:
x = trial.suggest_float("x", 0, 5)
y = trial.suggest_float("y", 1, 1)
return 4 * x**2 + 4 * y**2
study = create_study()
study.optimize(objective, n_trials=3)
param_importance = get_param_importances(study,
evaluator=evaluator_init_func())
assert len(param_importance) == 2
assert all([param in param_importance for param in ["x", "y"]])
assert param_importance["x"] > 0.0
assert param_importance["y"] == 0.0
def test_get_param_importances(storage_mode: str,
evaluator_init_func: Callable[
[], BaseImportanceEvaluator],
normalize: bool) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 0, 3, step=1)
x4 = trial.suggest_int("x4", -3, 3)
x5 = trial.suggest_int("x5", 1, 5, log=True)
x6 = trial.suggest_categorical("x6", [1.0, 1.1, 1.2])
if trial.number % 2 == 0:
# Conditional parameters are ignored unless `params` is specified and is not `None`.
x7 = trial.suggest_float("x7", 0.1, 3)
assert isinstance(x6, float)
value = x1**4 + x2 + x3 - x4**2 - x5 + x6
if trial.number % 2 == 0:
value += x7
return value
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=samplers.RandomSampler())
study.optimize(objective, n_trials=3)
param_importance = get_param_importances(
study, evaluator=evaluator_init_func(), normalize=normalize)
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == 6
assert all(param_name in param_importance
for param_name in ["x1", "x2", "x3", "x4", "x5", "x6"])
prev_importance = float("inf")
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
assert importance <= prev_importance
prev_importance = importance
# Sanity check for param importances
assert all(0 <= x < float("inf") for x in param_importance.values())
if normalize:
assert np.isclose(sum(param_importance.values()), 1.0)
def test_get_param_importances_with_target(storage_mode: str,
evaluator_init_func: Callable[
[], BaseImportanceEvaluator],
normalize: bool) -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 0, 3, step=1)
if trial.number % 2 == 0:
x4 = trial.suggest_float("x4", 0.1, 3)
value = x1**4 + x2 + x3
if trial.number % 2 == 0:
value += x4
return value
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage)
study.optimize(objective, n_trials=3)
param_importance = get_param_importances(
study,
evaluator=evaluator_init_func(),
target=lambda t: t.params["x1"] + t.params["x2"],
normalize=normalize,
)
assert isinstance(param_importance, OrderedDict)
assert len(param_importance) == 3
assert all(param_name in param_importance
for param_name in ["x1", "x2", "x3"])
prev_importance = float("inf")
for param_name, importance in param_importance.items():
assert isinstance(param_name, str)
assert isinstance(importance, float)
assert importance <= prev_importance
prev_importance = importance
# Sanity check for param importances
assert all(0 <= x < float("inf") for x in param_importance.values())
if normalize:
assert np.isclose(sum(param_importance.values()), 1.0)
def test_get_param_importances_invalid_no_completed_trials_params() -> None:
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
if trial.number % 2 == 0:
_ = trial.suggest_loguniform("x2", 0.1, 3)
raise optuna.exceptions.TrialPruned
return x1 ** 2
study = create_study()
study.optimize(objective, n_trials=3)
# None of the trials with `x2` are completed.
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x2"])
# None of the trials with `x2` are completed. Adding "x1" should not matter.
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x1", "x2"])
# None of the trials contain `x3`.
with pytest.raises(ValueError):
get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x3"])
layerin = Input(name='input', shape=(X.shape[-1],))
hidden = Dense(trial.suggest_int('n_hidden_units_enc1', 10, 64, 8), activation=act)(layerin)
hidden = Dense(trial.suggest_int(f'n_hidden_units_enc2', 4, 64, 8), activation=act)(hidden)
dense = Dense(name='encoded', units=2, activation='linear')(hidden) # tanh oder linear or relu or elu all work ok-ish
layerout = Dense(units=X.shape[-1])(dense) # (hidden_decode)
model = Model(layerin, layerout)
model.compile(trial.suggest_categorical('optim', ['adam', 'rmsprop', 'sgd']), 'mean_squared_error')
model.fit(x=X, y=X, epochs=150, batch_size=32, callbacks=[EarlyStopping(patience=5, monitor='loss'), TFKerasPruningCallback(trial, 'loss')])
return MeanSquaredError()(X, model.predict(X))
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=10)
#%%
study.trials_dataframe().sort_values('value')
# %%
from optuna.visualization import plot_parallel_coordinate
plot_parallel_coordinate(study)
#%%
from optuna.importance import get_param_importances
get_param_importances(study)
#%%
import hiplot as hip
hip.Experiment.from_dataframe(study.trials_dataframe()).display()
