def test_get_param_importance_target_is_none_and_study_is_multi_obj( storage_mode: str, evaluator_init_func: Callable[[], BaseImportanceEvaluator], ) -> None: def objective(trial: Trial) -> Tuple[float, float]: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 0, 3, step=1) x4 = trial.suggest_int("x4", -3, 3) x5 = trial.suggest_int("x5", 1, 5, log=True) x6 = trial.suggest_categorical("x6", [1.0, 1.1, 1.2]) if trial.number % 2 == 0: # Conditional parameters are ignored unless `params` is specified and is not `None`. x7 = trial.suggest_float("x7", 0.1, 3) assert isinstance(x6, float) value = x1**4 + x2 + x3 - x4**2 - x5 + x6 if trial.number % 2 == 0: value += x7 return value, 0.0 with StorageSupplier(storage_mode) as storage: study = create_study(directions=["minimize", "minimize"], storage=storage) study.optimize(objective, n_trials=3) with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func())
def test_get_param_importances_unnormalized_experimental() -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) return x1**2 study = create_study() study.optimize(objective, n_trials=4) with pytest.warns(ExperimentalWarning): get_param_importances(study, normalize=False)
def test_get_param_importances_invalid_dynamic_search_space_params() -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_uniform("x1", 0.1, trial.number + 0.1) return x1 ** 2 study = create_study() study.optimize(objective, n_trials=3) with pytest.raises(ValueError): get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x1"])
def test_get_param_importances_invalid_single_trial() -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_uniform("x1", 0.1, 3) return x1 ** 2 study = create_study() study.optimize(objective, n_trials=1) with pytest.raises(ValueError): get_param_importances(study, evaluator=FanovaImportanceEvaluator())
def test_get_param_importances_invalid_evaluator_type() -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) return x1 ** 2 study = create_study() study.optimize(objective, n_trials=3) with pytest.raises(TypeError): get_param_importances(study, evaluator={}) # type: ignore
def test_get_param_importances_invalid_single_trial( evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) return x1**2 study = create_study() study.optimize(objective, n_trials=1) with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func())
def test_get_param_importances_invalid_empty_study( evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None: study = create_study() with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func()) study.optimize(pruned_objective, n_trials=3) with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func())
def test_get_param_importances_invalid_dynamic_search_space_params( evaluator_init_func: Callable[[], BaseImportanceEvaluator] ) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, trial.number + 0.1) return x1 ** 2 study = create_study() study.optimize(objective, n_trials=3) with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func(), params=["x1"])
def test_get_param_importances_invalid_empty_study() -> None: study = create_study() with pytest.raises(ValueError): get_param_importances(study, evaluator=FanovaImportanceEvaluator()) def objective(trial: Trial) -> float: raise optuna.exceptions.TrialPruned study.optimize(objective, n_trials=3) with pytest.raises(ValueError): get_param_importances(study, evaluator=FanovaImportanceEvaluator())
def test_get_param_importances_invalid_params_type( evaluator_init_func: Callable[[], BaseImportanceEvaluator] ) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) return x1 ** 2 study = create_study() study.optimize(objective, n_trials=3) with pytest.raises(TypeError): get_param_importances(study, evaluator=evaluator_init_func(), params={}) # type: ignore with pytest.raises(TypeError): get_param_importances(study, evaluator=evaluator_init_func(), params=[0]) # type: ignore
def test_get_param_importances_with_params( storage_init_func: Callable[[], storages.BaseStorage], params: List[str], evaluator_init_func: Callable[[], BaseImportanceEvaluator], ) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 0, 3, step=1) if trial.number % 2 == 0: x4 = trial.suggest_float("x4", 0.1, 3) value = x1 ** 4 + x2 + x3 if trial.number % 2 == 0: value += x4 return value study = create_study(storage_init_func()) study.optimize(objective, n_trials=10) param_importance = get_param_importances(study, evaluator=evaluator_init_func(), params=params) assert isinstance(param_importance, OrderedDict) assert len(param_importance) == len(params) assert all(param in param_importance for param in params) for param_name, importance in param_importance.items(): assert isinstance(param_name, str) assert isinstance(importance, float) if len(param_importance) > 0: assert math.isclose(1.0, sum(i for i in param_importance.values()), abs_tol=1e-5)
def test_get_param_importances(storage_init_func: Callable[[], storages.BaseStorage]) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_uniform("x1", 0.1, 3) x2 = trial.suggest_loguniform("x2", 0.1, 3) x3 = trial.suggest_discrete_uniform("x3", 0, 3, 1) x4 = trial.suggest_int("x4", -3, 3) x5 = trial.suggest_categorical("x5", [1.0, 1.1, 1.2]) if trial.number % 2 == 0: # Conditional parameters are ignored unless `params` is specified and is not `None`. x6 = trial.suggest_uniform("x6", 0.1, 3) assert isinstance(x5, float) value = x1 ** 4 + x2 + x3 - x4 ** 2 - x5 if trial.number % 2 == 0: value += x6 return value study = create_study(storage_init_func(), sampler=samplers.RandomSampler()) study.optimize(objective, n_trials=3) param_importance = get_param_importances(study, evaluator=FanovaImportanceEvaluator()) assert isinstance(param_importance, OrderedDict) assert len(param_importance) == 5 assert all(param_name in param_importance for param_name in ["x1", "x2", "x3", "x4", "x5"]) for param_name, importance in param_importance.items(): assert isinstance(param_name, str) assert isinstance(importance, float) assert math.isclose(1.0, sum(i for i in param_importance.values()))
def test_get_param_importances_with_params( storage_mode: str, params: List[str], evaluator_init_func: Callable[[], BaseImportanceEvaluator], ) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 0, 3, step=1) if trial.number % 2 == 0: x4 = trial.suggest_float("x4", 0.1, 3) value = x1**4 + x2 + x3 if trial.number % 2 == 0: value += x4 return value with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage) study.optimize(objective, n_trials=10) param_importance = get_param_importances( study, evaluator=evaluator_init_func(), params=params) assert isinstance(param_importance, OrderedDict) assert len(param_importance) == len(params) assert all(param in param_importance for param in params) for param_name, importance in param_importance.items(): assert isinstance(param_name, str) assert isinstance(importance, float) # Sanity check for param importances assert all(0 <= x < float("inf") for x in param_importance.values())
def test_get_param_importances_with_target( storage_init_func: Callable[[], storages.BaseStorage], evaluator_init_func: Callable[[], BaseImportanceEvaluator], ) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 0, 3, step=1) if trial.number % 2 == 0: x4 = trial.suggest_float("x4", 0.1, 3) value = x1 ** 4 + x2 + x3 if trial.number % 2 == 0: value += x4 return value study = create_study(storage_init_func()) study.optimize(objective, n_trials=3) param_importance = get_param_importances( study, evaluator=evaluator_init_func(), target=lambda t: t.params["x1"] + t.params["x2"], ) assert isinstance(param_importance, OrderedDict) assert len(param_importance) == 3 assert all(param_name in param_importance for param_name in ["x1", "x2", "x3"]) prev_importance = float("inf") for param_name, importance in param_importance.items(): assert isinstance(param_name, str) assert isinstance(importance, float) assert importance <= prev_importance prev_importance = importance assert math.isclose(1.0, sum(param_importance.values()), abs_tol=1e-5)
def plot_param_importances( study: Study, evaluator = None, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, target_name: str = "Objective Value", ): _imports.check() _check_plot_args(study, target, target_name) layout = go.Layout( title="Hyperparameter Importances", xaxis={"title": f"Importance for {target_name}"}, yaxis={"title": "Hyperparameter"}, showlegend=False, ) # Importances cannot be evaluated without completed trials. # Return an empty figure for consistency with other visualization functions. trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE] if len(trials) == 0: logger.warning("Study instance does not contain completed trials.") return go.Figure(data=[], layout=layout) if evaluator is None: evaluator = ImportanceEvaluator() try: importances, importance_paras = get_param_importances( study, evaluator=evaluator, params=params, target=target ) except RuntimeError: # sometimes it is returning error e.g. when number of trials are < 4 return None, None, None importances = OrderedDict(reversed(list(importances.items()))) importance_values = list(importances.values()) param_names = list(importances.keys()) fig = go.Figure( data=[ go.Bar( x=importance_values, y=param_names, text=importance_values, texttemplate="%{text:.2f}", textposition="outside", cliponaxis=False, # Ensure text is not clipped. hovertemplate=[ _make_hovertext(param_name, importance, study) for param_name, importance in importances.items() ], marker_color=[_get_color(param_name, study) for param_name in param_names], orientation="h", ) ], layout=layout, ) return importances, importance_paras, fig
def test_get_param_importances_invalid_no_completed_trials_params( evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) if trial.number % 2 == 0: _ = trial.suggest_float("x2", 0.1, 3, log=True) raise optuna.TrialPruned return x1**2 study = create_study() study.optimize(objective, n_trials=3) # None of the trials with `x2` are completed. with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func(), params=["x2"]) # None of the trials with `x2` are completed. Adding "x1" should not matter. with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func(), params=["x1", "x2"]) # None of the trials contain `x3`. with pytest.raises(ValueError): get_param_importances(study, evaluator=evaluator_init_func(), params=["x3"])
def save_metrics(self): trials_importance: OrderedDict = get_param_importances(self.study) with open( "./data_processing/data/results/" + self.study_name + "_importance.csv", "w") as csv_file: w = csv.writer(csv_file) for key, val in trials_importance.items(): w.writerow([key, val]) trails_df = self.study.trials_dataframe() trails_df.to_csv("./data_processing/data/results/" + self.study_name + "_history.csv", mode='w', index=False)
def test_get_param_importances_empty_search_space( evaluator_init_func: Callable[[], BaseImportanceEvaluator]) -> None: def objective(trial: Trial) -> float: x = trial.suggest_float("x", 0, 5) y = trial.suggest_float("y", 1, 1) return 4 * x**2 + 4 * y**2 study = create_study() study.optimize(objective, n_trials=3) param_importance = get_param_importances(study, evaluator=evaluator_init_func()) assert len(param_importance) == 2 assert all([param in param_importance for param in ["x", "y"]]) assert param_importance["x"] > 0.0 assert param_importance["y"] == 0.0
def test_get_param_importances(storage_mode: str, evaluator_init_func: Callable[ [], BaseImportanceEvaluator], normalize: bool) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 0, 3, step=1) x4 = trial.suggest_int("x4", -3, 3) x5 = trial.suggest_int("x5", 1, 5, log=True) x6 = trial.suggest_categorical("x6", [1.0, 1.1, 1.2]) if trial.number % 2 == 0: # Conditional parameters are ignored unless `params` is specified and is not `None`. x7 = trial.suggest_float("x7", 0.1, 3) assert isinstance(x6, float) value = x1**4 + x2 + x3 - x4**2 - x5 + x6 if trial.number % 2 == 0: value += x7 return value with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage, sampler=samplers.RandomSampler()) study.optimize(objective, n_trials=3) param_importance = get_param_importances( study, evaluator=evaluator_init_func(), normalize=normalize) assert isinstance(param_importance, OrderedDict) assert len(param_importance) == 6 assert all(param_name in param_importance for param_name in ["x1", "x2", "x3", "x4", "x5", "x6"]) prev_importance = float("inf") for param_name, importance in param_importance.items(): assert isinstance(param_name, str) assert isinstance(importance, float) assert importance <= prev_importance prev_importance = importance # Sanity check for param importances assert all(0 <= x < float("inf") for x in param_importance.values()) if normalize: assert np.isclose(sum(param_importance.values()), 1.0)
def test_get_param_importances_with_target(storage_mode: str, evaluator_init_func: Callable[ [], BaseImportanceEvaluator], normalize: bool) -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 0, 3, step=1) if trial.number % 2 == 0: x4 = trial.suggest_float("x4", 0.1, 3) value = x1**4 + x2 + x3 if trial.number % 2 == 0: value += x4 return value with StorageSupplier(storage_mode) as storage: study = create_study(storage=storage) study.optimize(objective, n_trials=3) param_importance = get_param_importances( study, evaluator=evaluator_init_func(), target=lambda t: t.params["x1"] + t.params["x2"], normalize=normalize, ) assert isinstance(param_importance, OrderedDict) assert len(param_importance) == 3 assert all(param_name in param_importance for param_name in ["x1", "x2", "x3"]) prev_importance = float("inf") for param_name, importance in param_importance.items(): assert isinstance(param_name, str) assert isinstance(importance, float) assert importance <= prev_importance prev_importance = importance # Sanity check for param importances assert all(0 <= x < float("inf") for x in param_importance.values()) if normalize: assert np.isclose(sum(param_importance.values()), 1.0)
def test_get_param_importances_invalid_no_completed_trials_params() -> None: def objective(trial: Trial) -> float: x1 = trial.suggest_uniform("x1", 0.1, 3) if trial.number % 2 == 0: _ = trial.suggest_loguniform("x2", 0.1, 3) raise optuna.exceptions.TrialPruned return x1 ** 2 study = create_study() study.optimize(objective, n_trials=3) # None of the trials with `x2` are completed. with pytest.raises(ValueError): get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x2"]) # None of the trials with `x2` are completed. Adding "x1" should not matter. with pytest.raises(ValueError): get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x1", "x2"]) # None of the trials contain `x3`. with pytest.raises(ValueError): get_param_importances(study, evaluator=FanovaImportanceEvaluator(), params=["x3"])
layerin = Input(name='input', shape=(X.shape[-1],)) hidden = Dense(trial.suggest_int('n_hidden_units_enc1', 10, 64, 8), activation=act)(layerin) hidden = Dense(trial.suggest_int(f'n_hidden_units_enc2', 4, 64, 8), activation=act)(hidden) dense = Dense(name='encoded', units=2, activation='linear')(hidden) # tanh oder linear or relu or elu all work ok-ish layerout = Dense(units=X.shape[-1])(dense) # (hidden_decode) model = Model(layerin, layerout) model.compile(trial.suggest_categorical('optim', ['adam', 'rmsprop', 'sgd']), 'mean_squared_error') model.fit(x=X, y=X, epochs=150, batch_size=32, callbacks=[EarlyStopping(patience=5, monitor='loss'), TFKerasPruningCallback(trial, 'loss')]) return MeanSquaredError()(X, model.predict(X)) study = optuna.create_study(direction='minimize') study.optimize(objective, n_trials=10) #%% study.trials_dataframe().sort_values('value') # %% from optuna.visualization import plot_parallel_coordinate plot_parallel_coordinate(study) #%% from optuna.importance import get_param_importances get_param_importances(study) #%% import hiplot as hip hip.Experiment.from_dataframe(study.trials_dataframe()).display()