def plot_param_importances(self, interactive=False): ''' Plot parameters importance. Parameters ---------- interactive : bool, optional Create & save to current wd interactive html plot. The default is False. Returns ------- None. ''' self._check_refit_status('plot_param_importance()') validate_plotting_interactive_argument(interactive) if interactive: from optuna.visualization import plot_param_importances fig = plot_param_importances(self._study) fig.write_html("param_importances_plot.html") try: self._display_html('param_importances_plot.html') except Exception as e: print(f'Display html error: {e}') print( f'Param Importances Plot is saved to {os.path.join(os.getcwd(), "param_importances_plot.html")}' ) else: from optuna.visualization.matplotlib import plot_param_importances import matplotlib.pyplot as plt plot_param_importances(self._study) plt.show()
def make_plots(logdir, study): logdir = f'{logdir}/plots' os.makedirs(logdir, exist_ok=True) plot_optimization_history(study).write_image(f'{logdir}/history.svg') plot_intermediate_values(study).write_image(f'{logdir}/intermediates.svg') plot_parallel_coordinate(study).write_image(f'{logdir}/parallel_coordinates.png') plot_slice(study).write_image(f'{logdir}/slices.svg') plot_param_importances(study).write_image(f'{logdir}/importances.svg')
def test_plot_param_importances() -> None: # Test with no trial. study = create_study() figure = plot_param_importances(study) assert len(figure.data) == 0 study = prepare_study_with_trials(with_c_d=True) # Test with a trial. figure = plot_param_importances(study) assert len(figure.data) == 1 assert set(figure.data[0].y) == set( ("param_b", "param_d")) # "param_a", "param_c" are conditional. assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5) assert figure.layout.xaxis.title.text == "Importance for Objective Value" # Test with an evaluator. plot_param_importances(study, evaluator=MeanDecreaseImpurityImportanceEvaluator()) assert len(figure.data) == 1 assert set(figure.data[0].y) == set( ("param_b", "param_d")) # "param_a", "param_c" are conditional. assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5) # Test with a trial to select parameter. figure = plot_param_importances(study, params=["param_b"]) assert len(figure.data) == 1 assert figure.data[0].y == ("param_b", ) assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5) # Test with a customized target value. with pytest.warns(UserWarning): figure = plot_param_importances( study, target=lambda t: t.params["param_b"] + t.params["param_d"]) assert len(figure.data) == 1 assert set(figure.data[0].y) == set( ("param_b", "param_d")) # "param_a", "param_c" are conditional. assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5) # Test with a customized target name. figure = plot_param_importances(study, target_name="Target Name") assert figure.layout.xaxis.title.text == "Importance for Target Name" # Test with wrong parameters. with pytest.raises(ValueError): plot_param_importances(study, params=["optuna"]) # Ignore failed trials. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError, )) figure = plot_param_importances(study) assert len(figure.data) == 0
def test_multi_objective_trial_with_infinite_value_ignored( target_idx: int, inf_value: float, evaluator: BaseImportanceEvaluator, n_trial: int) -> None: def _multi_objective_function(trial: Trial) -> Tuple[float, float]: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 2, 4, log=True) return x1, x2 * x3 seed = 13 study = create_study(directions=["minimize", "minimize"], sampler=RandomSampler(seed=seed)) study.optimize(_multi_objective_function, n_trials=n_trial) # A figure is created without a trial with an inf value. figure_without_inf = plot_param_importances( study, evaluator=evaluator, target=lambda t: t.values[target_idx]) # A trial with an inf value is added into the study manually. study.add_trial( create_trial( values=[inf_value, inf_value], params={ "x1": 1.0, "x2": 1.0, "x3": 3.0 }, distributions={ "x1": FloatDistribution(low=0.1, high=3), "x2": FloatDistribution(low=0.1, high=3, log=True), "x3": FloatDistribution(low=2, high=4, log=True), }, )) # A figure is created with a trial with an inf value. figure_with_inf = plot_param_importances( study, evaluator=evaluator, target=lambda t: t.values[target_idx]) # Obtained figures should be the same between with inf and without inf, # because the last trial whose objective value is an inf is ignored. assert figure_without_inf == figure_with_inf
def test_trial_with_infinite_value_ignored(inf_value: int, evaluator: BaseImportanceEvaluator, n_trial: int) -> None: def _objective(trial: Trial) -> float: x1 = trial.suggest_float("x1", 0.1, 3) x2 = trial.suggest_float("x2", 0.1, 3, log=True) x3 = trial.suggest_float("x3", 2, 4, log=True) return x1 + x2 * x3 seed = 13 study = create_study(sampler=RandomSampler(seed=seed)) study.optimize(_objective, n_trials=n_trial) # A figure is created without a trial with an inf value. figure_without_inf = plot_param_importances(study, evaluator=evaluator) # A trial with an inf value is added into the study manually. study.add_trial( create_trial( value=inf_value, params={ "x1": 1.0, "x2": 1.0, "x3": 3.0 }, distributions={ "x1": FloatDistribution(low=0.1, high=3), "x2": FloatDistribution(low=0.1, high=3, log=True), "x3": FloatDistribution(low=2, high=4, log=True), }, )) # A figure is created with a trial with an inf value. figure_with_inf = plot_param_importances(study, evaluator=evaluator) # Obtained figures should be the same between with inf and without inf, # because the last trial whose objective value is an inf is ignored. assert figure_without_inf == figure_with_inf
def test_switch_label_when_param_insignificant() -> None: def _objective(trial: Trial) -> int: x = trial.suggest_int("x", 0, 2) _ = trial.suggest_int("y", -1, 1) return x**2 study = create_study() for x in range(1, 3): study.enqueue_trial({"x": x, "y": 0}) study.optimize(_objective, n_trials=2) figure = plot_param_importances(study) # Test if label for `y` param has been switched to `<0.01`. labels = figure.data[0].text assert labels == ("<0.01", "1.00")
def test_plot_param_importances() -> None: # Test with no trial. study = create_study() figure = plot_param_importances(study) assert len(figure.data) == 0 study = prepare_study_with_trials(with_c_d=True) # Test with a trial. figure = plot_param_importances(study) assert len(figure.data) == 1 assert set(figure.data[0].y) == set( ("param_b", "param_d")) # "param_a", "param_c" are conditional. assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5) # Test with an evaluator. plot_param_importances(study, evaluator=MeanDecreaseImpurityImportanceEvaluator()) assert len(figure.data) == 1 assert set(figure.data[0].y) == set( ("param_b", "param_d")) # "param_a", "param_c" are conditional. assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5) # Test with a trial to select parameter. figure = plot_param_importances(study, params=["param_b"]) assert len(figure.data) == 1 assert figure.data[0].y == ("param_b", ) assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5) # Test with wrong parameters. with pytest.raises(ValueError): plot_param_importances(study, params=["optuna"]) # Ignore failed trials. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError, )) figure = plot_param_importances(study) assert len(figure.data) == 0
# Visualize high-dimensional parameter relationships. See :func:`~optuna.visualization.plot_parallel_coordinate` for the details. plot_parallel_coordinate(study) ################################################################################################### # Select parameters to visualize. plot_parallel_coordinate(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize hyperparameter relationships. See :func:`~optuna.visualization.plot_contour` for the details. plot_contour(study) ################################################################################################### # Select parameters to visualize. plot_contour(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize individual hyperparameters as slice plot. See :func:`~optuna.visualization.plot_slice` for the details. plot_slice(study) ################################################################################################### # Select parameters to visualize. plot_slice(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize parameter importances. See :func:`~optuna.visualization.plot_param_importances` for the details. plot_param_importances(study) ################################################################################################### # Visualize empirical distribution function. See :func:`~optuna.visualization.plot_edf` for the details. plot_edf(study)
study = optuna.create_study(direction="maximize", pruner=optuna.pruners.MedianPruner()) study.optimize(objective, n_trials=100, timeout=600) # Visualize the optimization history. plot_optimization_history(study).show() # Visualize the learning curves of the trials. plot_intermediate_values(study).show() # Visualize high-dimensional parameter relationships. plot_parallel_coordinate(study).show() # Select parameters to visualize. plot_parallel_coordinate(study, params=["lr_init", "n_units_l0"]).show() # Visualize hyperparameter relationships. plot_contour(study).show() # Select parameters to visualize. plot_contour(study, params=["n_units_l0", "n_units_l1"]).show() # Visualize individual hyperparameters. plot_slice(study).show() # Select parameters to visualize. plot_slice(study, params=["n_units_l0", "n_units_l1"]).show() # Visualize parameter importances. plot_param_importances(study).show()
timeout=TIMEOUT) except KeyboardInterrupt: pass print("Number of finished trials: ", len(study.trials)) print("Best trial:") trial = study.best_trial print(f" Value: {trial.value}") print(" Params: ") for key, value in trial.params.items(): print(f" {key}: {value}") print(" User attrs:") for key, value in trial.user_attrs.items(): print(f" {key}: {value}") # Write report study.trials_dataframe().to_csv("study_results_a2c_cartpole.csv") with open("study.pkl", "wb+") as f: pkl.dump(study, f) fig1 = plot_optimization_history(study) fig2 = plot_param_importances(study) fig1.show() fig2.show()
def test_target_is_none_and_study_is_multi_obj() -> None: study = create_study(directions=["minimize", "minimize"]) with pytest.raises(ValueError): plot_param_importances(study)
def hyperparameters_optimization(self) -> None: if self.verbose > 0: print("Optimizing hyperparameters") if self.storage is not None and self.study_name is None: warnings.warn( f"You passed a remote storage: {self.storage} but no `--study-name`." "The study name will be generated by Optuna, make sure to re-use the same study name " "when you want to do distributed hyperparameter optimization.") if self.tensorboard_log is not None: warnings.warn( "Tensorboard log is deactivated when running hyperparameter optimization" ) self.tensorboard_log = None # TODO: eval each hyperparams several times to account for noisy evaluation sampler = self._create_sampler(self.sampler) pruner = self._create_pruner(self.pruner) if self.verbose > 0: print(f"Sampler: {self.sampler} - Pruner: {self.pruner}") study = optuna.create_study( sampler=sampler, pruner=pruner, storage=self.storage, study_name=self.study_name, load_if_exists=True, direction="maximize", ) try: study.optimize(self.objective, n_trials=self.n_trials, n_jobs=self.n_jobs) except KeyboardInterrupt: pass print("Number of finished trials: ", len(study.trials)) print("Best trial:") trial = study.best_trial print("Value: ", trial.value) print("Params: ") for key, value in trial.params.items(): print(f" {key}: {value}") report_name = ( f"report_{self.env_id}_{self.n_trials}-trials-{self.n_timesteps}" f"-{self.sampler}-{self.pruner}_{int(time.time())}") log_path = os.path.join(self.log_folder, self.algo, report_name) if self.verbose: print(f"Writing report to {log_path}") # Write report os.makedirs(os.path.dirname(log_path), exist_ok=True) study.trials_dataframe().to_csv(f"{log_path}.csv") # Save python object to inspect/re-use it later with open(f"{log_path}.pkl", "wb+") as f: pkl.dump(study, f) # Skip plots if self.no_optim_plots: return # Plot optimization result try: fig1 = plot_optimization_history(study) fig2 = plot_param_importances(study) fig1.show() fig2.show() except (ValueError, ImportError, RuntimeError): pass
best_model_predict = best_model.predict(x_test) print('\nBest Model performance at competition:') print( 'RMSE: {:.4f} (should be lower than the trivial predictor using the mean MSE: {:.4f})' .format( math.sqrt(metrics.mean_squared_error(y_test, best_model_predict)), math.sqrt( metrics.mean_squared_error( y_test, [y_test.mean() for i in range(len(y_test))])))) print( 'R square: {:.4f} (should be higher than the trivial predictor using the mean: R square {:.4f})' .format( metrics.r2_score(y_test, best_model_predict), metrics.r2_score(y_test, [y_test.mean() for i in range(len(y_test))]))) #3.8 Final model train best_model.fit(x, y) y_comp = [math.exp(i) for i in best_model.predict(x_comp)] submission = pd.DataFrame(columns=['Id', 'SalePrice']) submission['Id'] = pd.Series(range(1461, 2920)) submission['SalePrice'] = pd.Series(y_comp) submission.to_csv('submission.csv', index=False) #3.9 Optuna visualization ov.plot_optimization_history(knn_optuna).show() #ov.plot_parallel_coordinate(knn_optuna).show() ov.plot_contour(knn_optuna).show() ov.plot_slice(knn_optuna).show() ov.plot_param_importances(knn_optuna).show() #ov.plot_edf(knn_optuna).show()