def plot_param_importances(self, interactive=False):
'''
Plot parameters importance.
Parameters
----------
interactive : bool, optional
Create & save to current wd interactive html plot. The default is False.
Returns
-------
None.
'''
self._check_refit_status('plot_param_importance()')
validate_plotting_interactive_argument(interactive)
if interactive:
from optuna.visualization import plot_param_importances
fig = plot_param_importances(self._study)
fig.write_html("param_importances_plot.html")
try:
self._display_html('param_importances_plot.html')
except Exception as e:
print(f'Display html error: {e}')
print(
f'Param Importances Plot is saved to {os.path.join(os.getcwd(), "param_importances_plot.html")}'
)
else:
from optuna.visualization.matplotlib import plot_param_importances
import matplotlib.pyplot as plt
plot_param_importances(self._study)
plt.show()
def make_plots(logdir, study):
logdir = f'{logdir}/plots'
os.makedirs(logdir, exist_ok=True)
plot_optimization_history(study).write_image(f'{logdir}/history.svg')
plot_intermediate_values(study).write_image(f'{logdir}/intermediates.svg')
plot_parallel_coordinate(study).write_image(f'{logdir}/parallel_coordinates.png')
plot_slice(study).write_image(f'{logdir}/slices.svg')
plot_param_importances(study).write_image(f'{logdir}/importances.svg')
def test_plot_param_importances() -> None:
# Test with no trial.
study = create_study()
figure = plot_param_importances(study)
assert len(figure.data) == 0
study = prepare_study_with_trials(with_c_d=True)
# Test with a trial.
figure = plot_param_importances(study)
assert len(figure.data) == 1
assert set(figure.data[0].y) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5)
assert figure.layout.xaxis.title.text == "Importance for Objective Value"
# Test with an evaluator.
plot_param_importances(study,
evaluator=MeanDecreaseImpurityImportanceEvaluator())
assert len(figure.data) == 1
assert set(figure.data[0].y) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5)
# Test with a trial to select parameter.
figure = plot_param_importances(study, params=["param_b"])
assert len(figure.data) == 1
assert figure.data[0].y == ("param_b", )
assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5)
# Test with a customized target value.
with pytest.warns(UserWarning):
figure = plot_param_importances(
study, target=lambda t: t.params["param_b"] + t.params["param_d"])
assert len(figure.data) == 1
assert set(figure.data[0].y) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5)
# Test with a customized target name.
figure = plot_param_importances(study, target_name="Target Name")
assert figure.layout.xaxis.title.text == "Importance for Target Name"
# Test with wrong parameters.
with pytest.raises(ValueError):
plot_param_importances(study, params=["optuna"])
# Ignore failed trials.
def fail_objective(_: Trial) -> float:
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError, ))
figure = plot_param_importances(study)
assert len(figure.data) == 0
def test_multi_objective_trial_with_infinite_value_ignored(
target_idx: int, inf_value: float, evaluator: BaseImportanceEvaluator,
n_trial: int) -> None:
def _multi_objective_function(trial: Trial) -> Tuple[float, float]:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 2, 4, log=True)
return x1, x2 * x3
seed = 13
study = create_study(directions=["minimize", "minimize"],
sampler=RandomSampler(seed=seed))
study.optimize(_multi_objective_function, n_trials=n_trial)
# A figure is created without a trial with an inf value.
figure_without_inf = plot_param_importances(
study, evaluator=evaluator, target=lambda t: t.values[target_idx])
# A trial with an inf value is added into the study manually.
study.add_trial(
create_trial(
values=[inf_value, inf_value],
params={
"x1": 1.0,
"x2": 1.0,
"x3": 3.0
},
distributions={
"x1": FloatDistribution(low=0.1, high=3),
"x2": FloatDistribution(low=0.1, high=3, log=True),
"x3": FloatDistribution(low=2, high=4, log=True),
},
))
# A figure is created with a trial with an inf value.
figure_with_inf = plot_param_importances(
study, evaluator=evaluator, target=lambda t: t.values[target_idx])
# Obtained figures should be the same between with inf and without inf,
# because the last trial whose objective value is an inf is ignored.
assert figure_without_inf == figure_with_inf
def test_trial_with_infinite_value_ignored(inf_value: int,
evaluator: BaseImportanceEvaluator,
n_trial: int) -> None:
def _objective(trial: Trial) -> float:
x1 = trial.suggest_float("x1", 0.1, 3)
x2 = trial.suggest_float("x2", 0.1, 3, log=True)
x3 = trial.suggest_float("x3", 2, 4, log=True)
return x1 + x2 * x3
seed = 13
study = create_study(sampler=RandomSampler(seed=seed))
study.optimize(_objective, n_trials=n_trial)
# A figure is created without a trial with an inf value.
figure_without_inf = plot_param_importances(study, evaluator=evaluator)
# A trial with an inf value is added into the study manually.
study.add_trial(
create_trial(
value=inf_value,
params={
"x1": 1.0,
"x2": 1.0,
"x3": 3.0
},
distributions={
"x1": FloatDistribution(low=0.1, high=3),
"x2": FloatDistribution(low=0.1, high=3, log=True),
"x3": FloatDistribution(low=2, high=4, log=True),
},
))
# A figure is created with a trial with an inf value.
figure_with_inf = plot_param_importances(study, evaluator=evaluator)
# Obtained figures should be the same between with inf and without inf,
# because the last trial whose objective value is an inf is ignored.
assert figure_without_inf == figure_with_inf
def test_switch_label_when_param_insignificant() -> None:
def _objective(trial: Trial) -> int:
x = trial.suggest_int("x", 0, 2)
_ = trial.suggest_int("y", -1, 1)
return x**2
study = create_study()
for x in range(1, 3):
study.enqueue_trial({"x": x, "y": 0})
study.optimize(_objective, n_trials=2)
figure = plot_param_importances(study)
# Test if label for `y` param has been switched to `<0.01`.
labels = figure.data[0].text
assert labels == ("<0.01", "1.00")
def test_plot_param_importances() -> None:
# Test with no trial.
study = create_study()
figure = plot_param_importances(study)
assert len(figure.data) == 0
study = prepare_study_with_trials(with_c_d=True)
# Test with a trial.
figure = plot_param_importances(study)
assert len(figure.data) == 1
assert set(figure.data[0].y) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5)
# Test with an evaluator.
plot_param_importances(study,
evaluator=MeanDecreaseImpurityImportanceEvaluator())
assert len(figure.data) == 1
assert set(figure.data[0].y) == set(
("param_b", "param_d")) # "param_a", "param_c" are conditional.
assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5)
# Test with a trial to select parameter.
figure = plot_param_importances(study, params=["param_b"])
assert len(figure.data) == 1
assert figure.data[0].y == ("param_b", )
assert math.isclose(1.0, sum(i for i in figure.data[0].x), abs_tol=1e-5)
# Test with wrong parameters.
with pytest.raises(ValueError):
plot_param_importances(study, params=["optuna"])
# Ignore failed trials.
def fail_objective(_: Trial) -> float:
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError, ))
figure = plot_param_importances(study)
assert len(figure.data) == 0
# Visualize high-dimensional parameter relationships. See :func:`~optuna.visualization.plot_parallel_coordinate` for the details. plot_parallel_coordinate(study) ################################################################################################### # Select parameters to visualize. plot_parallel_coordinate(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize hyperparameter relationships. See :func:`~optuna.visualization.plot_contour` for the details. plot_contour(study) ################################################################################################### # Select parameters to visualize. plot_contour(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize individual hyperparameters as slice plot. See :func:`~optuna.visualization.plot_slice` for the details. plot_slice(study) ################################################################################################### # Select parameters to visualize. plot_slice(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize parameter importances. See :func:`~optuna.visualization.plot_param_importances` for the details. plot_param_importances(study) ################################################################################################### # Visualize empirical distribution function. See :func:`~optuna.visualization.plot_edf` for the details. plot_edf(study)
study = optuna.create_study(direction="maximize",
pruner=optuna.pruners.MedianPruner())
study.optimize(objective, n_trials=100, timeout=600)
# Visualize the optimization history.
plot_optimization_history(study).show()
# Visualize the learning curves of the trials.
plot_intermediate_values(study).show()
# Visualize high-dimensional parameter relationships.
plot_parallel_coordinate(study).show()
# Select parameters to visualize.
plot_parallel_coordinate(study, params=["lr_init", "n_units_l0"]).show()
# Visualize hyperparameter relationships.
plot_contour(study).show()
# Select parameters to visualize.
plot_contour(study, params=["n_units_l0", "n_units_l1"]).show()
# Visualize individual hyperparameters.
plot_slice(study).show()
# Select parameters to visualize.
plot_slice(study, params=["n_units_l0", "n_units_l1"]).show()
# Visualize parameter importances.
plot_param_importances(study).show()
timeout=TIMEOUT)
except KeyboardInterrupt:
pass
print("Number of finished trials: ", len(study.trials))
print("Best trial:")
trial = study.best_trial
print(f" Value: {trial.value}")
print(" Params: ")
for key, value in trial.params.items():
print(f" {key}: {value}")
print(" User attrs:")
for key, value in trial.user_attrs.items():
print(f" {key}: {value}")
# Write report
study.trials_dataframe().to_csv("study_results_a2c_cartpole.csv")
with open("study.pkl", "wb+") as f:
pkl.dump(study, f)
fig1 = plot_optimization_history(study)
fig2 = plot_param_importances(study)
fig1.show()
fig2.show()
def test_target_is_none_and_study_is_multi_obj() -> None:
study = create_study(directions=["minimize", "minimize"])
with pytest.raises(ValueError):
plot_param_importances(study)
def hyperparameters_optimization(self) -> None:
if self.verbose > 0:
print("Optimizing hyperparameters")
if self.storage is not None and self.study_name is None:
warnings.warn(
f"You passed a remote storage: {self.storage} but no `--study-name`."
"The study name will be generated by Optuna, make sure to re-use the same study name "
"when you want to do distributed hyperparameter optimization.")
if self.tensorboard_log is not None:
warnings.warn(
"Tensorboard log is deactivated when running hyperparameter optimization"
)
self.tensorboard_log = None
# TODO: eval each hyperparams several times to account for noisy evaluation
sampler = self._create_sampler(self.sampler)
pruner = self._create_pruner(self.pruner)
if self.verbose > 0:
print(f"Sampler: {self.sampler} - Pruner: {self.pruner}")
study = optuna.create_study(
sampler=sampler,
pruner=pruner,
storage=self.storage,
study_name=self.study_name,
load_if_exists=True,
direction="maximize",
)
try:
study.optimize(self.objective,
n_trials=self.n_trials,
n_jobs=self.n_jobs)
except KeyboardInterrupt:
pass
print("Number of finished trials: ", len(study.trials))
print("Best trial:")
trial = study.best_trial
print("Value: ", trial.value)
print("Params: ")
for key, value in trial.params.items():
print(f" {key}: {value}")
report_name = (
f"report_{self.env_id}_{self.n_trials}-trials-{self.n_timesteps}"
f"-{self.sampler}-{self.pruner}_{int(time.time())}")
log_path = os.path.join(self.log_folder, self.algo, report_name)
if self.verbose:
print(f"Writing report to {log_path}")
# Write report
os.makedirs(os.path.dirname(log_path), exist_ok=True)
study.trials_dataframe().to_csv(f"{log_path}.csv")
# Save python object to inspect/re-use it later
with open(f"{log_path}.pkl", "wb+") as f:
pkl.dump(study, f)
# Skip plots
if self.no_optim_plots:
return
# Plot optimization result
try:
fig1 = plot_optimization_history(study)
fig2 = plot_param_importances(study)
fig1.show()
fig2.show()
except (ValueError, ImportError, RuntimeError):
pass
best_model_predict = best_model.predict(x_test)
print('\nBest Model performance at competition:')
print(
'RMSE: {:.4f} (should be lower than the trivial predictor using the mean MSE: {:.4f})'
.format(
math.sqrt(metrics.mean_squared_error(y_test, best_model_predict)),
math.sqrt(
metrics.mean_squared_error(
y_test, [y_test.mean() for i in range(len(y_test))]))))
print(
'R square: {:.4f} (should be higher than the trivial predictor using the mean: R square {:.4f})'
.format(
metrics.r2_score(y_test, best_model_predict),
metrics.r2_score(y_test, [y_test.mean() for i in range(len(y_test))])))
#3.8 Final model train
best_model.fit(x, y)
y_comp = [math.exp(i) for i in best_model.predict(x_comp)]
submission = pd.DataFrame(columns=['Id', 'SalePrice'])
submission['Id'] = pd.Series(range(1461, 2920))
submission['SalePrice'] = pd.Series(y_comp)
submission.to_csv('submission.csv', index=False)
#3.9 Optuna visualization
ov.plot_optimization_history(knn_optuna).show()
#ov.plot_parallel_coordinate(knn_optuna).show()
ov.plot_contour(knn_optuna).show()
ov.plot_slice(knn_optuna).show()
ov.plot_param_importances(knn_optuna).show()
#ov.plot_edf(knn_optuna).show()
