def test_color_map(direction: str) -> None:
study = prepare_study_with_trials(with_c_d=False, direction=direction)
# `target` is `None`.
line = plot_parallel_coordinate(study).data[0]["line"]
assert COLOR_SCALE == [v[1] for v in line["colorscale"]]
if direction == "minimize":
assert line["reversescale"]
else:
assert not line["reversescale"]
# When `target` is not `None`, `reversescale` is always `True`.
line = plot_parallel_coordinate(study,
target=lambda t: t.number).data[0]["line"]
assert COLOR_SCALE == [v[1] for v in line["colorscale"]]
assert line["reversescale"]
# Multi-objective optimization.
study = prepare_study_with_trials(with_c_d=False,
n_objectives=2,
direction=direction)
line = plot_parallel_coordinate(study,
target=lambda t: t.number).data[0]["line"]
assert COLOR_SCALE == [v[1] for v in line["colorscale"]]
assert line["reversescale"]
def make_plots(logdir, study):
logdir = f'{logdir}/plots'
os.makedirs(logdir, exist_ok=True)
plot_optimization_history(study).write_image(f'{logdir}/history.svg')
plot_intermediate_values(study).write_image(f'{logdir}/intermediates.svg')
plot_parallel_coordinate(study).write_image(f'{logdir}/parallel_coordinates.png')
plot_slice(study).write_image(f'{logdir}/slices.svg')
plot_param_importances(study).write_image(f'{logdir}/importances.svg')
def test_plot_parallel_coordinate_unique_hyper_param() -> None:
# Test case when one unique value is suggested during the optimization.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"param_b": 30
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": FloatDistribution(1, 1000, log=True),
},
))
# Both hyperparameters contain unique values.
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 0.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, )
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
assert figure.data[0]["dimensions"][1]["values"] == (0.0, )
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (math.log10(30),
math.log10(30))
assert figure.data[0]["dimensions"][2]["values"] == (math.log10(30), )
assert figure.data[0]["dimensions"][2]["ticktext"] == ("30", )
assert figure.data[0]["dimensions"][2]["tickvals"] == (math.log10(30), )
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "preferred",
"param_b": 20
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"param_b": FloatDistribution(1, 1000, log=True),
},
))
# Still "category_a" contains unique suggested value during the optimization.
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 0)
assert figure.data[0]["dimensions"][1]["values"] == (0.0, 0.0)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", )
assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
def main(cfg):
# stratified_kfold(cfg)
USE_KFOLD = cfg.values.val_args.use_kfold
if USE_KFOLD:
objective = lambda trial: hp_search(trial, cfg)
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=20)
best_trial = study.best_trial
print(f'best acc : {best_trial.values}')
print(f'best acc : {best_trial.params}')
# params = {
# 'batch_size': 1,
# 'lr': 1.61e-06,
# 'dropout': 0.633
# }
scores = 0
for j in range(5):
scr = train(j, best_trial.params, cfg,
save_model=True) #best_trial.params
scores += scr
print(scores / 5)
df = study.trials_dataframe(attrs=("number", "value", "params",
"state"))
df.to_csv('./hpo_result.csv')
plot_parallel_coordinate(study)
else:
# train_df = whole_df[whole_df.kfold != 0].reset_index(drop=True)
# valid_df = whole_df[whole_df.kfold == 0].reset_index(drop=True)
# tokenized_train = tokenized_dataset(train_df, tokenizer)
# tokenized_val = tokenized_dataset(valid_df, tokenizer)
# RE_train_dataset = RE_Dataset(tokenized_train, train_df['label'].values)
# RE_val_dataset = RE_Dataset(tokenized_val, valid_df['label'].values)
# dataset = {
# 'train': RE_train_dataset,
# 'valid': RE_val_dataset
# }
# objective = lambda trial: hp_search(trial,
# model_name=MODEL_NAME,
# dataset=dataset,
# label_nbr=42,
# metric_name='accuracy',
# device=device)
# study = optuna.create_study(direction='maximize')
# study.optimize(objective, timeout=1800)
pass
def test_plot_parallel_coordinate_only_missing_params() -> None:
# When all trials contain only a part of parameters,
# the plot returns an empty figure.
study = create_study()
study.add_trial(
create_trial(
value=0.0,
params={"param_a": 1e-6},
distributions={
"param_a": FloatDistribution(1e-7, 1e-2, log=True),
},
)
)
study.add_trial(
create_trial(
value=1.0,
params={"param_b": 200},
distributions={
"param_b": FloatDistribution(1, 1000, log=True),
},
)
)
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
def __call__(self, study, trial):
import optuna.visualization as vis
self.exp.log_metric('run_score', trial.value)
self.exp.log_metric('best_so_far_run_score', study.best_value)
self.exp.log_text('run_parameters', str(trial.params))
if self.log_study:
pickle_and_log_artifact(study, 'study.pkl', experiment=self.exp)
if self.log_optimization_history:
log_chart(name='optimization_history',
chart=vis.plot_optimization_history(study),
experiment=self.exp)
if self.log_contour:
log_chart(name='contour',
chart=vis.plot_contour(study, params=self.params),
experiment=self.exp)
if self.log_parallel_coordinate:
log_chart(name='parallel_coordinate',
chart=vis.plot_parallel_coordinate(study,
params=self.params),
experiment=self.exp)
if self.log_slice:
log_chart(name='slice',
chart=vis.plot_slice(study, params=self.params),
experiment=self.exp)
def test_plot_parallel_coordinate_categorical_params() -> None:
# Test with categorical params that cannot be converted to numeral.
study_categorical_params = create_study()
distributions: Dict[str, BaseDistribution] = {
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
}
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={"category_a": "preferred", "category_b": "net"},
distributions=distributions,
)
)
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={"category_a": "opt", "category_b": "una"},
distributions=distributions,
)
)
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 1)
assert figure.data[0]["dimensions"][1]["values"] == (0, 1)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", "opt")
assert figure.data[0]["dimensions"][2]["label"] == "category_b"
assert figure.data[0]["dimensions"][2]["range"] == (0, 1)
assert figure.data[0]["dimensions"][2]["values"] == (0, 1)
assert figure.data[0]["dimensions"][2]["ticktext"] == ("net", "una")
def test_plot_parallel_coordinate_log_params() -> None:
# Test with log params.
study_log_params = create_study()
study_log_params.add_trial(
create_trial(
value=0.0,
params={
"param_a": 1e-6,
"param_b": 10
},
distributions={
"param_a": FloatDistribution(1e-7, 1e-2, log=True),
"param_b": FloatDistribution(1, 1000, log=True),
},
))
study_log_params.add_trial(
create_trial(
value=1.0,
params={
"param_a": 2e-5,
"param_b": 200
},
distributions={
"param_a": FloatDistribution(1e-7, 1e-2, log=True),
"param_b": FloatDistribution(1, 1000, log=True),
},
))
study_log_params.add_trial(
create_trial(
value=0.1,
params={
"param_a": 1e-4,
"param_b": 30
},
distributions={
"param_a": FloatDistribution(1e-7, 1e-2, log=True),
"param_b": FloatDistribution(1, 1000, log=True),
},
))
figure = plot_parallel_coordinate(study_log_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0, 0.1)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (-6.0, -4.0)
assert figure.data[0]["dimensions"][1]["values"] == (-6, math.log10(2e-5),
-4)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("1e-06", "1e-05",
"0.0001")
assert figure.data[0]["dimensions"][1]["tickvals"] == (-6, -5, -4.0)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (1.0, math.log10(200))
assert figure.data[0]["dimensions"][2]["values"] == (1.0, math.log10(200),
math.log10(30))
assert figure.data[0]["dimensions"][2]["ticktext"] == ("10", "100", "200")
assert figure.data[0]["dimensions"][2]["tickvals"] == (1.0, 2.0,
math.log10(200))
def test_plot_parallel_coordinate_categorical_numeric_params() -> None:
# Test with categorical params that can be interpreted as numeric params.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": 2,
"category_b": 20
},
distributions={
"category_a": CategoricalDistribution((1, 2)),
"category_b": CategoricalDistribution((10, 20, 30)),
},
))
study_categorical_params.add_trial(
create_trial(
value=1.0,
params={
"category_a": 1,
"category_b": 30
},
distributions={
"category_a": CategoricalDistribution((1, 2)),
"category_b": CategoricalDistribution((10, 20, 30)),
},
))
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": 2,
"category_b": 10
},
distributions={
"category_a": CategoricalDistribution((1, 2)),
"category_b": CategoricalDistribution((10, 20, 30)),
},
))
# Trials are sorted by using param_a and param_b, i.e., trial#1, trial#2, and trial#0.
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (1.0, 2.0, 0.0)
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 1)
assert figure.data[0]["dimensions"][1]["values"] == (0, 1, 1)
assert figure.data[0]["dimensions"][1]["ticktext"] == (1, 2)
assert figure.data[0]["dimensions"][1]["tickvals"] == (0, 1)
assert figure.data[0]["dimensions"][2]["label"] == "category_b"
assert figure.data[0]["dimensions"][2]["range"] == (0, 2)
assert figure.data[0]["dimensions"][2]["values"] == (2, 0, 1)
assert figure.data[0]["dimensions"][2]["ticktext"] == (10, 20, 30)
assert figure.data[0]["dimensions"][2]["tickvals"] == (0, 1, 2)
def log_study_info(study, experiment=None, log_charts=True, params=None):
"""Logs runs results and parameters to neptune.
Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric),
best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts
('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder.
Args:
study('optuna.study.Study'): Optuna study object after training is completed.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_charts('bool'): Whether optuna visualization charts should be logged. By default all charts are logged.
params(`list`): List of parameters to be visualized. Default is all parameters.
Examples:
Initialize neptune_monitor::
import neptune
import neptunecontrib.monitoring.optuna as opt_utils
neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME')
neptune.create_experiment(name='optuna sweep')
neptune_callback = opt_utils.NeptuneCallback()
Run Optuna training passing monitor as callback::
...
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100, callbacks=[neptune_callback])
opt_utils.log_study_info(study)
You can explore an example experiment in Neptune:
https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts
"""
import optuna.visualization as vis
_exp = experiment if experiment else neptune
_exp.log_metric('best_score', study.best_value)
_exp.set_property('best_parameters', study.best_params)
if log_charts:
log_chart(name='optimization_history',
chart=vis.plot_optimization_history(study),
experiment=_exp)
log_chart(name='contour',
chart=vis.plot_contour(study, params=params),
experiment=_exp)
log_chart(name='parallel_coordinate',
chart=vis.plot_parallel_coordinate(study, params=params),
experiment=_exp)
log_chart(name='slice',
chart=vis.plot_slice(study, params=params),
experiment=_exp)
pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
def test_color_map(direction: str) -> None:
study = create_study(direction=direction)
for i in range(3):
study.add_trial(
create_trial(
value=float(i),
params={"param_a": float(i), "param_b": float(i)},
distributions={
"param_a": FloatDistribution(0.0, 3.0),
"param_b": FloatDistribution(0.0, 3.0),
},
)
)
# `target` is `None`.
line = plot_parallel_coordinate(study).data[0]["line"]
assert COLOR_SCALE == [v[1] for v in line["colorscale"]]
if direction == "minimize":
assert line["reversescale"]
else:
assert not line["reversescale"]
# When `target` is not `None`, `reversescale` is always `True`.
line = plot_parallel_coordinate(study, target=lambda t: t.number).data[0]["line"]
assert COLOR_SCALE == [v[1] for v in line["colorscale"]]
assert line["reversescale"]
# Multi-objective optimization.
study = create_study(directions=[direction, direction])
for i in range(3):
study.add_trial(
create_trial(
values=[float(i), float(i)],
params={"param_a": float(i), "param_b": float(i)},
distributions={
"param_a": FloatDistribution(0.0, 3.0),
"param_b": FloatDistribution(0.0, 3.0),
},
)
)
line = plot_parallel_coordinate(study, target=lambda t: t.number).data[0]["line"]
assert COLOR_SCALE == [v[1] for v in line["colorscale"]]
assert line["reversescale"]
def draw_results(study):
# 优化历史
plt.figure()
fig = pv.plot_optimization_history(study)
fig.write_image("./output/opt_his.png")
plt.close()
# 等高线图
plt.figure()
fig = pv.plot_contour(study)
fig.write_image("./output/opt_contour.png")
plt.close()
# 经验分布图
plt.figure()
fig = pv.plot_edf(study)
fig.write_image("./output/opt_edf.png")
plt.close()
# 高维参数
plt.figure()
fig = pv.plot_parallel_coordinate(study)
fig.write_image("./output/opt_coordinate.png")
plt.close()
def test_plot_parallel_coordinate() -> None:
# Test with no trial.
study = create_study()
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
study = create_study(direction="minimize")
study.add_trial(
create_trial(
value=0.0,
params={"param_a": 1.0, "param_b": 2.0},
distributions={
"param_a": FloatDistribution(0.0, 3.0),
"param_b": FloatDistribution(0.0, 3.0),
},
)
)
study.add_trial(
create_trial(
value=2.0,
params={"param_b": 0.0},
distributions={"param_b": FloatDistribution(0.0, 3.0)},
)
)
study.add_trial(
create_trial(
value=1.0,
params={"param_a": 2.5, "param_b": 1.0},
distributions={
"param_a": FloatDistribution(0.0, 3.0),
"param_b": FloatDistribution(0.0, 3.0),
},
)
)
# Test with a trial.
figure = plot_parallel_coordinate(study)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0)
assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0)
# Test with a trial to select parameter.
figure = plot_parallel_coordinate(study, params=["param_a"])
assert len(figure.data[0]["dimensions"]) == 2
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
# Test with a customized target value.
with pytest.warns(UserWarning):
figure = plot_parallel_coordinate(
study, params=["param_a"], target=lambda t: t.params["param_b"]
)
assert len(figure.data[0]["dimensions"]) == 2
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (1.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (2.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
# Test with a customized target name.
figure = plot_parallel_coordinate(study, target_name="Target Name")
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Target Name"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0)
assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0)
# Test with wrong params that do not exist in trials
with pytest.raises(ValueError, match="Parameter optuna does not exist in your study."):
plot_parallel_coordinate(study, params=["optuna", "optuna"])
# Ignore failed trials.
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError,))
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
def test_target_is_none_and_study_is_multi_obj() -> None:
study = create_study(directions=["minimize", "minimize"])
with pytest.raises(ValueError):
plot_parallel_coordinate(study)
def test_plot_parallel_coordinate_with_categorical_numeric_params() -> None:
# Test with sample from multiple distributions including categorical params
# that can be interpreted as numeric params.
study_multi_distro_params = create_study()
study_multi_distro_params.add_trial(
create_trial(
value=0.0,
params={
"param_a": "preferred",
"param_b": 2,
"param_c": 30,
"param_d": 2
},
distributions={
"param_a": CategoricalDistribution(("preferred", "opt")),
"param_b": CategoricalDistribution((1, 2, 10)),
"param_c": FloatDistribution(1, 1000, log=True),
"param_d": CategoricalDistribution((1, -1, 2)),
},
))
study_multi_distro_params.add_trial(
create_trial(
value=1.0,
params={
"param_a": "opt",
"param_b": 1,
"param_c": 200,
"param_d": 2
},
distributions={
"param_a": CategoricalDistribution(("preferred", "opt")),
"param_b": CategoricalDistribution((1, 2, 10)),
"param_c": FloatDistribution(1, 1000, log=True),
"param_d": CategoricalDistribution((1, -1, 2)),
},
))
study_multi_distro_params.add_trial(
create_trial(
value=2.0,
params={
"param_a": "preferred",
"param_b": 10,
"param_c": 10,
"param_d": 1
},
distributions={
"param_a": CategoricalDistribution(("preferred", "opt")),
"param_b": CategoricalDistribution((1, 2, 10)),
"param_c": FloatDistribution(1, 1000, log=True),
"param_d": CategoricalDistribution((1, -1, 2)),
},
))
study_multi_distro_params.add_trial(
create_trial(
value=3.0,
params={
"param_a": "opt",
"param_b": 2,
"param_c": 10,
"param_d": -1
},
distributions={
"param_a": CategoricalDistribution(("preferred", "opt")),
"param_b": CategoricalDistribution((1, 2, 10)),
"param_c": FloatDistribution(1, 1000, log=True),
"param_d": CategoricalDistribution((-1, 1, 2)),
},
))
figure = plot_parallel_coordinate(study_multi_distro_params)
assert len(figure.data[0]["dimensions"]) == 5
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 3.0)
assert figure.data[0]["dimensions"][0]["values"] == (1.0, 3.0, 0.0, 2.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 1)
assert figure.data[0]["dimensions"][1]["values"] == (1, 1, 0, 0)
assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", "opt")
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (0, 2)
assert figure.data[0]["dimensions"][2]["values"] == (0, 1, 1, 2)
assert figure.data[0]["dimensions"][2]["ticktext"] == (1, 2, 10)
assert figure.data[0]["dimensions"][3]["label"] == "param_c"
assert figure.data[0]["dimensions"][3]["range"] == (1.0, math.log10(200))
assert figure.data[0]["dimensions"][3]["values"] == (math.log10(200), 1.0,
math.log10(30), 1.0)
assert figure.data[0]["dimensions"][3]["ticktext"] == ("10", "100", "200")
assert figure.data[0]["dimensions"][3]["tickvals"] == (1.0, 2.0,
math.log10(200))
assert figure.data[0]["dimensions"][4]["label"] == "param_d"
assert figure.data[0]["dimensions"][4]["range"] == (0, 2)
assert figure.data[0]["dimensions"][4]["values"] == (2, 0, 2, 1)
assert figure.data[0]["dimensions"][4]["ticktext"] == (-1, 1, 2)
def test_nonfinite_removed(value: float) -> None:
study = prepare_study_with_trials(value_for_first_trial=value)
figure = plot_parallel_coordinate(study)
assert all(np.isfinite(figure.data[0]["dimensions"][0]["values"]))
) study.optimize(objective, n_trials=100, timeout=600) ################################################################################################### # Plot functions # -------------- # Visualize the optimization history. See :func:`~optuna.visualization.plot_optimization_history` for the details. plot_optimization_history(study) ################################################################################################### # Visualize the learning curves of the trials. See :func:`~optuna.visualization.plot_intermediate_values` for the details. plot_intermediate_values(study) ################################################################################################### # Visualize high-dimensional parameter relationships. See :func:`~optuna.visualization.plot_parallel_coordinate` for the details. plot_parallel_coordinate(study) ################################################################################################### # Select parameters to visualize. plot_parallel_coordinate(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize hyperparameter relationships. See :func:`~optuna.visualization.plot_contour` for the details. plot_contour(study) ################################################################################################### # Select parameters to visualize. plot_contour(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize individual hyperparameters as slice plot. See :func:`~optuna.visualization.plot_slice` for the details.
def log_study_info(study, experiment=None,
log_study=True,
log_charts=True,
log_optimization_history=False,
log_contour=False,
log_parallel_coordinate=False,
log_slice=False,
params=None):
"""Logs runs results and parameters to neptune.
Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric),
best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts
('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder.
Args:
study('optuna.study.Study'): Optuna study object after training is completed.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_study('bool'): Whether optuna study object should be logged as pickle. Default is True.
log_charts('bool'): Deprecated argument. Whether all optuna visualizations charts should be logged.
By default all charts are sent.
To not log any charts set log_charts=False.
If you want to log a particular chart change the argument for that chart explicitly.
For example log_charts=False and log_slice=True will log only the slice plot to Neptune.
log_optimization_history('bool'): Whether optuna optimization history chart should be logged. Default is True.
log_contour('bool'): Whether optuna contour plot should be logged. Default is True.
log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. Default is True.
log_slice('bool'): Whether optuna slice chart should be logged. Default is True.
params(`list`): List of parameters to be visualized. Default is all parameters.
Examples:
Initialize neptune_monitor::
import neptune
import neptunecontrib.monitoring.optuna as opt_utils
neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME')
neptune.create_experiment(name='optuna sweep')
neptune_callback = opt_utils.NeptuneCallback()
Run Optuna training passing monitor as callback::
...
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100, callbacks=[neptune_callback])
opt_utils.log_study_info(study)
You can explore an example experiment in Neptune:
https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts
"""
import optuna.visualization as vis
_exp = experiment if experiment else neptune
_exp.log_metric('best_score', study.best_value)
_exp.set_property('best_parameters', study.best_params)
if log_charts:
message = """log_charts argument is depraceted and will be removed in future releases.
Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly.
"""
warnings.warn(message)
log_optimization_history = True
log_contour = True
log_parallel_coordinate = True
log_slice = True
if log_study:
pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
if log_optimization_history:
log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp)
if log_contour:
log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp)
if log_parallel_coordinate:
log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp)
if log_slice:
log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp)
def ml_mlp_mul_ms(station_name="종로구"):
print("Start Multivariate MLP Mean Seasonality Decomposition (MSE) Model")
targets = ["PM10", "PM25"]
# targets = ["SO2", "CO", "O3", "NO2", "PM10", "PM25",
# "temp", "u", "v", "pres", "humid", "prep", "snow"]
# 24*14 = 336
#sample_size = 336
sample_size = 48
output_size = 24
# If you want to debug, fast_dev_run = True and n_trials should be small number
fast_dev_run = False
n_trials = 128
# fast_dev_run = True
# n_trials = 1
# Hyper parameter
epoch_size = 500
batch_size = 64
learning_rate = 1e-3
# Blocked Cross Validation
# neglect small overlap between train_dates and valid_dates
# 11y = ((2y, 0.5y), (2y, 0.5y), (2y, 0.5y), (2.5y, 1y))
train_dates = [(dt.datetime(2008, 1, 4, 1).astimezone(SEOULTZ),
dt.datetime(2009, 12, 31, 23).astimezone(SEOULTZ)),
(dt.datetime(2010, 7, 1, 0).astimezone(SEOULTZ),
dt.datetime(2012, 6, 30, 23).astimezone(SEOULTZ)),
(dt.datetime(2013, 1, 1, 0).astimezone(SEOULTZ),
dt.datetime(2014, 12, 31, 23).astimezone(SEOULTZ)),
(dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ),
dt.datetime(2017, 12, 31, 23).astimezone(SEOULTZ))]
valid_dates = [(dt.datetime(2010, 1, 1, 0).astimezone(SEOULTZ),
dt.datetime(2010, 6, 30, 23).astimezone(SEOULTZ)),
(dt.datetime(2012, 7, 1, 0).astimezone(SEOULTZ),
dt.datetime(2012, 12, 31, 23).astimezone(SEOULTZ)),
(dt.datetime(2015, 1, 1, 0).astimezone(SEOULTZ),
dt.datetime(2015, 6, 30, 23).astimezone(SEOULTZ)),
(dt.datetime(2018, 1, 1, 0).astimezone(SEOULTZ),
dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ))]
train_valid_fdate = dt.datetime(2008, 1, 3, 1).astimezone(SEOULTZ)
train_valid_tdate = dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ)
# Debug
if fast_dev_run:
train_dates = [(dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ),
dt.datetime(2017, 12, 31, 23).astimezone(SEOULTZ))]
valid_dates = [(dt.datetime(2018, 1, 1, 0).astimezone(SEOULTZ),
dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ))]
train_valid_fdate = dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ)
train_valid_tdate = dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ)
test_fdate = dt.datetime(2019, 1, 1, 0).astimezone(SEOULTZ)
test_tdate = dt.datetime(2020, 10, 31, 23).astimezone(SEOULTZ)
# check date range assumption
assert len(train_dates) == len(valid_dates)
for i, (td, vd) in enumerate(zip(train_dates, valid_dates)):
assert vd[0] > td[1]
assert test_fdate > train_dates[-1][1]
assert test_fdate > valid_dates[-1][1]
train_features = [
"SO2", "CO", "NO2", "PM10", "PM25", "temp", "wind_spd", "wind_cdir",
"wind_sdir", "pres", "humid", "prep"
]
train_features_periodic = [
"SO2", "CO", "NO2", "PM10", "PM25", "temp", "wind_spd", "wind_cdir",
"wind_sdir", "pres", "humid"
]
train_features_nonperiodic = ["prep"]
for target in targets:
print("Training " + target + "...")
output_dir = Path(
f"/mnt/data/MLPMSMultivariate/{station_name}/{target}/")
Path.mkdir(output_dir, parents=True, exist_ok=True)
model_dir = output_dir / "models"
Path.mkdir(model_dir, parents=True, exist_ok=True)
log_dir = output_dir / "log"
Path.mkdir(log_dir, parents=True, exist_ok=True)
_df_h = data.load_imputed(HOURLY_DATA_PATH)
df_h = _df_h.query('stationCode == "' +
str(SEOUL_STATIONS[station_name]) + '"')
if station_name == '종로구' and \
not Path("/input/python/input_jongno_imputed_hourly_pandas.csv").is_file():
# load imputed result
df_h.to_csv("/input/python/input_jongno_imputed_hourly_pandas.csv")
# construct dataset for seasonality
print("Construct Train/Validation Sets...", flush=True)
train_valid_dataset = construct_dataset(train_valid_fdate,
train_valid_tdate,
filepath=HOURLY_DATA_PATH,
station_name=station_name,
target=target,
sample_size=sample_size,
output_size=output_size,
transform=False)
# compute seasonality
train_valid_dataset.preprocess()
# For Block Cross Validation..
# load dataset in given range dates and transform using scaler from train_valid_set
# all dataset are saved in tuple
print("Construct Training Sets...", flush=True)
train_datasets = tuple(
construct_dataset(td[0],
td[1],
scaler_X=train_valid_dataset.scaler_X,
scaler_Y=train_valid_dataset.scaler_Y,
filepath=HOURLY_DATA_PATH,
station_name=station_name,
target=target,
sample_size=sample_size,
output_size=output_size,
features=train_features,
features_periodic=train_features_periodic,
features_nonperiodic=train_features_nonperiodic,
transform=True) for td in train_dates)
print("Construct Validation Sets...", flush=True)
valid_datasets = tuple(
construct_dataset(vd[0],
vd[1],
scaler_X=train_valid_dataset.scaler_X,
scaler_Y=train_valid_dataset.scaler_Y,
filepath=HOURLY_DATA_PATH,
station_name=station_name,
target=target,
sample_size=sample_size,
output_size=output_size,
features=train_features,
features_periodic=train_features_periodic,
features_nonperiodic=train_features_nonperiodic,
transform=True) for vd in valid_dates)
# just single test set
print("Construct Test Sets...", flush=True)
test_dataset = construct_dataset(
test_fdate,
test_tdate,
scaler_X=train_valid_dataset.scaler_X,
scaler_Y=train_valid_dataset.scaler_Y,
filepath=HOURLY_DATA_PATH,
station_name=station_name,
target=target,
sample_size=sample_size,
output_size=output_size,
features=train_features,
features_periodic=train_features_periodic,
features_nonperiodic=train_features_nonperiodic,
transform=True)
# convert tuple of datasets to ConcatDataset
train_dataset = ConcatDataset(train_datasets)
val_dataset = ConcatDataset(valid_datasets)
# num_layer == number of hidden layer
hparams = Namespace(num_layers=1,
layer_size=128,
learning_rate=learning_rate,
batch_size=batch_size)
def objective(trial):
model = BaseMLPModel(
trial=trial,
hparams=hparams,
input_size=sample_size * len(train_features),
sample_size=sample_size,
output_size=output_size,
station_name=station_name,
target=target,
features=train_features,
features_periodic=train_features_periodic,
features_nonperiodic=train_features_nonperiodic,
train_dataset=train_dataset,
val_dataset=val_dataset,
test_dataset=test_dataset,
scaler_X=train_valid_dataset.scaler_X,
scaler_Y=train_valid_dataset.scaler_Y,
output_dir=output_dir)
# most basic trainer, uses good defaults
trainer = Trainer(gpus=1 if torch.cuda.is_available() else None,
precision=32,
min_epochs=1,
max_epochs=20,
default_root_dir=output_dir,
fast_dev_run=fast_dev_run,
logger=True,
checkpoint_callback=False,
callbacks=[
PyTorchLightningPruningCallback(
trial, monitor="valid/MSE")
])
trainer.fit(model)
# Don't Log
# hyperparameters = model.hparams
# trainer.logger.log_hyperparams(hyperparameters)
return trainer.callback_metrics.get("valid/MSE")
if n_trials > 1:
study = optuna.create_study(direction="minimize")
study.enqueue_trial({
'sigma': 1.3,
'num_layers': 4,
'layer_size': 8,
'learning_rate': learning_rate,
'batch_size': batch_size
})
study.enqueue_trial({
'sigma': 1.3,
'num_layers': 4,
'layer_size': 32,
'learning_rate': learning_rate,
'batch_size': batch_size
})
study.enqueue_trial({
'sigma': 1.3,
'num_layers': 4,
'layer_size': 64,
'learning_rate': learning_rate,
'batch_size': batch_size
})
study.enqueue_trial({
'sigma': 1.3,
'num_layers': 4,
'layer_size': 32,
'learning_rate': learning_rate,
'batch_size': batch_size
})
study.enqueue_trial({
'sigma': 1.3,
'num_layers': 8,
'layer_size': 32,
'learning_rate': learning_rate,
'batch_size': batch_size
})
study.enqueue_trial({
'sigma': 1.3,
'num_layers': 12,
'layer_size': 32,
'learning_rate': learning_rate,
'batch_size': batch_size
})
study.enqueue_trial({
'sigma': 0.7,
'num_layers': 4,
'layer_size': 32,
'learning_rate': learning_rate,
'batch_size': batch_size
})
study.enqueue_trial({
'sigma': 2.0,
'num_layers': 4,
'layer_size': 32,
'learning_rate': learning_rate,
'batch_size': batch_size
})
# timeout = 3600*36 = 36h
study.optimize(objective, n_trials=n_trials, timeout=3600 * 36)
trial = study.best_trial
print(" Value: ", trial.value)
print(" Params: ")
for key, value in trial.params.items():
print(" {}: {}".format(key, value))
print("sample_size : ", sample_size)
print("output_size : ", output_size)
# plot optmization results
fig_cont1 = optv.plot_contour(study,
params=['num_layers', 'layer_size'])
fig_cont1.write_image(
str(output_dir / "contour_num_layers_layer_size.png"))
fig_cont1.write_image(
str(output_dir / "contour_num_layers_layer_size.svg"))
fig_edf = optv.plot_edf(study)
fig_edf.write_image(str(output_dir / "edf.png"))
fig_edf.write_image(str(output_dir / "edf.svg"))
fig_iv = optv.plot_intermediate_values(study)
fig_iv.write_image(str(output_dir / "intermediate_values.png"))
fig_iv.write_image(str(output_dir / "intermediate_values.svg"))
fig_his = optv.plot_optimization_history(study)
fig_his.write_image(str(output_dir / "opt_history.png"))
fig_his.write_image(str(output_dir / "opt_history.svg"))
fig_pcoord = optv.plot_parallel_coordinate(
study, params=['num_layers', 'layer_size'])
fig_pcoord.write_image(str(output_dir / "parallel_coord.png"))
fig_pcoord.write_image(str(output_dir / "parallel_coord.svg"))
fig_slice = optv.plot_slice(study,
params=['num_layers', 'layer_size'])
fig_slice.write_image(str(output_dir / "slice.png"))
fig_slice.write_image(str(output_dir / "slice.svg"))
# set hparams with optmized value
hparams.num_layers = trial.params['num_layers']
hparams.layer_size = trial.params['layer_size']
dict_hparams = copy.copy(vars(hparams))
dict_hparams["sample_size"] = sample_size
dict_hparams["output_size"] = output_size
with open(output_dir / 'hparams.json', 'w') as f:
print(dict_hparams, file=f)
with open(output_dir / 'hparams.csv', 'w') as f:
print(pd.DataFrame.from_dict(dict_hparams, orient='index'),
file=f)
model = BaseMLPModel(hparams=hparams,
input_size=sample_size * len(train_features),
sample_size=sample_size,
output_size=output_size,
station_name=station_name,
target=target,
features=train_features,
features_periodic=train_features_periodic,
features_nonperiodic=train_features_nonperiodic,
train_dataset=train_dataset,
val_dataset=val_dataset,
test_dataset=test_dataset,
scaler_X=train_valid_dataset.scaler_X,
scaler_Y=train_valid_dataset.scaler_Y,
output_dir=output_dir)
# record input
for i, _train_set in enumerate(train_datasets):
_train_set.to_csv(
model.data_dir /
("df_trainset_{0}_".format(str(i).zfill(2)) + target + ".csv"))
for i, _valid_set in enumerate(valid_datasets):
_valid_set.to_csv(
model.data_dir /
("df_validset_{0}_".format(str(i).zfill(2)) + target + ".csv"))
train_valid_dataset.to_csv(model.data_dir /
("df_trainvalidset_" + target + ".csv"))
test_dataset.to_csv(model.data_dir / ("df_testset_" + target + ".csv"))
checkpoint_callback = pl.callbacks.ModelCheckpoint(os.path.join(
model_dir, "train_{epoch}_{valid/MSE:.2f}"),
monitor="valid/MSE",
period=10)
early_stop_callback = EarlyStopping(monitor='valid/MSE',
min_delta=0.001,
patience=30,
verbose=True,
mode='min')
log_version = dt.date.today().strftime("%y%m%d-%H-%M")
loggers = [ \
TensorBoardLogger(log_dir, version=log_version),
CSVLogger(log_dir, version=log_version)]
# most basic trainer, uses good defaults
trainer = Trainer(gpus=1 if torch.cuda.is_available() else None,
precision=32,
min_epochs=1,
max_epochs=epoch_size,
default_root_dir=output_dir,
fast_dev_run=fast_dev_run,
logger=loggers,
log_every_n_steps=5,
flush_logs_every_n_steps=10,
callbacks=[early_stop_callback],
checkpoint_callback=checkpoint_callback)
trainer.fit(model)
# run test set
trainer.test(ckpt_path=None)
shutil.rmtree(model_dir)
def show_parallel_coordinates(self, params=None):
return visualization.plot_parallel_coordinate(self.study,
params=params)
layerin = Input(name='input', shape=(X.shape[-1],))
hidden = Dense(trial.suggest_int('n_hidden_units_enc1', 10, 64, 8), activation=act)(layerin)
hidden = Dense(trial.suggest_int(f'n_hidden_units_enc2', 4, 64, 8), activation=act)(hidden)
dense = Dense(name='encoded', units=2, activation='linear')(hidden) # tanh oder linear or relu or elu all work ok-ish
layerout = Dense(units=X.shape[-1])(dense) # (hidden_decode)
model = Model(layerin, layerout)
model.compile(trial.suggest_categorical('optim', ['adam', 'rmsprop', 'sgd']), 'mean_squared_error')
model.fit(x=X, y=X, epochs=150, batch_size=32, callbacks=[EarlyStopping(patience=5, monitor='loss'), TFKerasPruningCallback(trial, 'loss')])
return MeanSquaredError()(X, model.predict(X))
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=10)
#%%
study.trials_dataframe().sort_values('value')
# %%
from optuna.visualization import plot_parallel_coordinate
plot_parallel_coordinate(study)
#%%
from optuna.importance import get_param_importances
get_param_importances(study)
#%%
import hiplot as hip
hip.Experiment.from_dataframe(study.trials_dataframe()).display()
if __name__ == "__main__":
study = optuna.create_study(direction="maximize",
pruner=optuna.pruners.MedianPruner())
study.optimize(objective, n_trials=100, timeout=600)
# Visualize the optimization history.
plot_optimization_history(study).show()
# Visualize the learning curves of the trials.
plot_intermediate_values(study).show()
# Visualize high-dimensional parameter relationships.
plot_parallel_coordinate(study).show()
# Select parameters to visualize.
plot_parallel_coordinate(study, params=["lr_init", "n_units_l0"]).show()
# Visualize hyperparameter relationships.
plot_contour(study).show()
# Select parameters to visualize.
plot_contour(study, params=["n_units_l0", "n_units_l1"]).show()
# Visualize individual hyperparameters.
plot_slice(study).show()
# Select parameters to visualize.
plot_slice(study, params=["n_units_l0", "n_units_l1"]).show()
def test_plot_parallel_coordinate() -> None:
# Test with no trial.
study = create_study()
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
study = prepare_study_with_trials(with_c_d=False)
# Test with a trial.
figure = plot_parallel_coordinate(study)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0)
assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0)
# Test with a trial to select parameter.
figure = plot_parallel_coordinate(study, params=["param_a"])
assert len(figure.data[0]["dimensions"]) == 2
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
# Test with a customized target value.
with pytest.warns(UserWarning):
figure = plot_parallel_coordinate(study,
params=["param_a"],
target=lambda t: t.params["param_b"])
assert len(figure.data[0]["dimensions"]) == 2
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (1.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (2.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
# Test with a customized target name.
figure = plot_parallel_coordinate(study, target_name="Target Name")
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Target Name"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0)
assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0)
# Test with wrong params that do not exist in trials
with pytest.raises(ValueError,
match="Parameter optuna does not exist in your study."):
plot_parallel_coordinate(study, params=["optuna", "optuna"])
# Ignore failed trials.
def fail_objective(_: Trial) -> float:
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError, ))
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
def test_plot_parallel_coordinate() -> None:
# Test with no trial.
study = create_study()
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
study = prepare_study_with_trials(with_c_d=False)
# Test with a trial.
figure = plot_parallel_coordinate(study)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][2]["values"] == (2.0, 0.0, 1.0)
# Test with a trial to select parameter.
figure = plot_parallel_coordinate(study, params=["param_a"])
assert len(figure.data[0]["dimensions"]) == 2
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
# Test with a customized target value.
figure = plot_parallel_coordinate(study,
params=["param_a"],
target=lambda t: t.params["param_b"])
assert len(figure.data[0]["dimensions"]) == 2
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (2.0, 0.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
# Test with a customized target name.
figure = plot_parallel_coordinate(study, target_name="Target Name")
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Target Name"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0)
assert figure.data[0]["dimensions"][1]["label"] == "param_a"
assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5)
assert figure.data[0]["dimensions"][2]["label"] == "param_b"
assert figure.data[0]["dimensions"][2]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][2]["values"] == (2.0, 0.0, 1.0)
# Test with wrong params that do not exist in trials
with pytest.raises(ValueError,
match="Parameter optuna does not exist in your study."):
plot_parallel_coordinate(study, params=["optuna", "optuna"])
# Ignore failed trials.
def fail_objective(_: Trial) -> float:
raise ValueError
study = create_study()
study.optimize(fail_objective, n_trials=1, catch=(ValueError, ))
figure = plot_parallel_coordinate(study)
assert len(figure.data) == 0
# Test with categorical params that cannot be converted to numeral.
study_categorical_params = create_study()
study_categorical_params.add_trial(
create_trial(
value=0.0,
params={
"category_a": "preferred",
"category_b": "net"
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
))
study_categorical_params.add_trial(
create_trial(
value=2.0,
params={
"category_a": "opt",
"category_b": "una"
},
distributions={
"category_a": CategoricalDistribution(("preferred", "opt")),
"category_b": CategoricalDistribution(("net", "una")),
},
))
figure = plot_parallel_coordinate(study_categorical_params)
assert len(figure.data[0]["dimensions"]) == 3
assert figure.data[0]["dimensions"][0]["label"] == "Objective Value"
assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0)
assert figure.data[0]["dimensions"][1]["label"] == "category_a"
assert figure.data[0]["dimensions"][1]["range"] == (0, 1)
assert figure.data[0]["dimensions"][1]["values"] == (0, 1)
assert figure.data[0]["dimensions"][1]["ticktext"] == (["preferred",
0], ["opt", 1])
assert figure.data[0]["dimensions"][2]["label"] == "category_b"
assert figure.data[0]["dimensions"][2]["range"] == (0, 1)
assert figure.data[0]["dimensions"][2]["values"] == (0, 1)
assert figure.data[0]["dimensions"][2]["ticktext"] == (["net",
0], ["una", 1])
def test_nonfinite_multiobjective(objective: int, value: float) -> None:
study = prepare_study_with_trials(n_objectives=2, value_for_first_trial=value)
figure = plot_parallel_coordinate(study, target=lambda t: t.values[objective])
assert all(np.isfinite(figure.data[0]["dimensions"][0]["values"]))
