def test_color_map(direction: str) -> None: study = prepare_study_with_trials(with_c_d=False, direction=direction) # `target` is `None`. line = plot_parallel_coordinate(study).data[0]["line"] assert COLOR_SCALE == [v[1] for v in line["colorscale"]] if direction == "minimize": assert line["reversescale"] else: assert not line["reversescale"] # When `target` is not `None`, `reversescale` is always `True`. line = plot_parallel_coordinate(study, target=lambda t: t.number).data[0]["line"] assert COLOR_SCALE == [v[1] for v in line["colorscale"]] assert line["reversescale"] # Multi-objective optimization. study = prepare_study_with_trials(with_c_d=False, n_objectives=2, direction=direction) line = plot_parallel_coordinate(study, target=lambda t: t.number).data[0]["line"] assert COLOR_SCALE == [v[1] for v in line["colorscale"]] assert line["reversescale"]
def make_plots(logdir, study): logdir = f'{logdir}/plots' os.makedirs(logdir, exist_ok=True) plot_optimization_history(study).write_image(f'{logdir}/history.svg') plot_intermediate_values(study).write_image(f'{logdir}/intermediates.svg') plot_parallel_coordinate(study).write_image(f'{logdir}/parallel_coordinates.png') plot_slice(study).write_image(f'{logdir}/slices.svg') plot_param_importances(study).write_image(f'{logdir}/importances.svg')
def test_plot_parallel_coordinate_unique_hyper_param() -> None: # Test case when one unique value is suggested during the optimization. study_categorical_params = create_study() study_categorical_params.add_trial( create_trial( value=0.0, params={ "category_a": "preferred", "param_b": 30 }, distributions={ "category_a": CategoricalDistribution(("preferred", "opt")), "param_b": FloatDistribution(1, 1000, log=True), }, )) # Both hyperparameters contain unique values. figure = plot_parallel_coordinate(study_categorical_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 0.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, ) assert figure.data[0]["dimensions"][1]["label"] == "category_a" assert figure.data[0]["dimensions"][1]["range"] == (0, 0) assert figure.data[0]["dimensions"][1]["values"] == (0.0, ) assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", ) assert figure.data[0]["dimensions"][1]["tickvals"] == (0, ) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (math.log10(30), math.log10(30)) assert figure.data[0]["dimensions"][2]["values"] == (math.log10(30), ) assert figure.data[0]["dimensions"][2]["ticktext"] == ("30", ) assert figure.data[0]["dimensions"][2]["tickvals"] == (math.log10(30), ) study_categorical_params.add_trial( create_trial( value=2.0, params={ "category_a": "preferred", "param_b": 20 }, distributions={ "category_a": CategoricalDistribution(("preferred", "opt")), "param_b": FloatDistribution(1, 1000, log=True), }, )) # Still "category_a" contains unique suggested value during the optimization. figure = plot_parallel_coordinate(study_categorical_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][1]["label"] == "category_a" assert figure.data[0]["dimensions"][1]["range"] == (0, 0) assert figure.data[0]["dimensions"][1]["values"] == (0.0, 0.0) assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", ) assert figure.data[0]["dimensions"][1]["tickvals"] == (0, )
def main(cfg): # stratified_kfold(cfg) USE_KFOLD = cfg.values.val_args.use_kfold if USE_KFOLD: objective = lambda trial: hp_search(trial, cfg) study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=20) best_trial = study.best_trial print(f'best acc : {best_trial.values}') print(f'best acc : {best_trial.params}') # params = { # 'batch_size': 1, # 'lr': 1.61e-06, # 'dropout': 0.633 # } scores = 0 for j in range(5): scr = train(j, best_trial.params, cfg, save_model=True) #best_trial.params scores += scr print(scores / 5) df = study.trials_dataframe(attrs=("number", "value", "params", "state")) df.to_csv('./hpo_result.csv') plot_parallel_coordinate(study) else: # train_df = whole_df[whole_df.kfold != 0].reset_index(drop=True) # valid_df = whole_df[whole_df.kfold == 0].reset_index(drop=True) # tokenized_train = tokenized_dataset(train_df, tokenizer) # tokenized_val = tokenized_dataset(valid_df, tokenizer) # RE_train_dataset = RE_Dataset(tokenized_train, train_df['label'].values) # RE_val_dataset = RE_Dataset(tokenized_val, valid_df['label'].values) # dataset = { # 'train': RE_train_dataset, # 'valid': RE_val_dataset # } # objective = lambda trial: hp_search(trial, # model_name=MODEL_NAME, # dataset=dataset, # label_nbr=42, # metric_name='accuracy', # device=device) # study = optuna.create_study(direction='maximize') # study.optimize(objective, timeout=1800) pass
def test_plot_parallel_coordinate_only_missing_params() -> None: # When all trials contain only a part of parameters, # the plot returns an empty figure. study = create_study() study.add_trial( create_trial( value=0.0, params={"param_a": 1e-6}, distributions={ "param_a": FloatDistribution(1e-7, 1e-2, log=True), }, ) ) study.add_trial( create_trial( value=1.0, params={"param_b": 200}, distributions={ "param_b": FloatDistribution(1, 1000, log=True), }, ) ) figure = plot_parallel_coordinate(study) assert len(figure.data) == 0
def __call__(self, study, trial): import optuna.visualization as vis self.exp.log_metric('run_score', trial.value) self.exp.log_metric('best_so_far_run_score', study.best_value) self.exp.log_text('run_parameters', str(trial.params)) if self.log_study: pickle_and_log_artifact(study, 'study.pkl', experiment=self.exp) if self.log_optimization_history: log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=self.exp) if self.log_contour: log_chart(name='contour', chart=vis.plot_contour(study, params=self.params), experiment=self.exp) if self.log_parallel_coordinate: log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=self.params), experiment=self.exp) if self.log_slice: log_chart(name='slice', chart=vis.plot_slice(study, params=self.params), experiment=self.exp)
def test_plot_parallel_coordinate_categorical_params() -> None: # Test with categorical params that cannot be converted to numeral. study_categorical_params = create_study() distributions: Dict[str, BaseDistribution] = { "category_a": CategoricalDistribution(("preferred", "opt")), "category_b": CategoricalDistribution(("net", "una")), } study_categorical_params.add_trial( create_trial( value=0.0, params={"category_a": "preferred", "category_b": "net"}, distributions=distributions, ) ) study_categorical_params.add_trial( create_trial( value=2.0, params={"category_a": "opt", "category_b": "una"}, distributions=distributions, ) ) figure = plot_parallel_coordinate(study_categorical_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0) assert figure.data[0]["dimensions"][1]["label"] == "category_a" assert figure.data[0]["dimensions"][1]["range"] == (0, 1) assert figure.data[0]["dimensions"][1]["values"] == (0, 1) assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", "opt") assert figure.data[0]["dimensions"][2]["label"] == "category_b" assert figure.data[0]["dimensions"][2]["range"] == (0, 1) assert figure.data[0]["dimensions"][2]["values"] == (0, 1) assert figure.data[0]["dimensions"][2]["ticktext"] == ("net", "una")
def test_plot_parallel_coordinate_log_params() -> None: # Test with log params. study_log_params = create_study() study_log_params.add_trial( create_trial( value=0.0, params={ "param_a": 1e-6, "param_b": 10 }, distributions={ "param_a": FloatDistribution(1e-7, 1e-2, log=True), "param_b": FloatDistribution(1, 1000, log=True), }, )) study_log_params.add_trial( create_trial( value=1.0, params={ "param_a": 2e-5, "param_b": 200 }, distributions={ "param_a": FloatDistribution(1e-7, 1e-2, log=True), "param_b": FloatDistribution(1, 1000, log=True), }, )) study_log_params.add_trial( create_trial( value=0.1, params={ "param_a": 1e-4, "param_b": 30 }, distributions={ "param_a": FloatDistribution(1e-7, 1e-2, log=True), "param_b": FloatDistribution(1, 1000, log=True), }, )) figure = plot_parallel_coordinate(study_log_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0, 0.1) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (-6.0, -4.0) assert figure.data[0]["dimensions"][1]["values"] == (-6, math.log10(2e-5), -4) assert figure.data[0]["dimensions"][1]["ticktext"] == ("1e-06", "1e-05", "0.0001") assert figure.data[0]["dimensions"][1]["tickvals"] == (-6, -5, -4.0) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (1.0, math.log10(200)) assert figure.data[0]["dimensions"][2]["values"] == (1.0, math.log10(200), math.log10(30)) assert figure.data[0]["dimensions"][2]["ticktext"] == ("10", "100", "200") assert figure.data[0]["dimensions"][2]["tickvals"] == (1.0, 2.0, math.log10(200))
def test_plot_parallel_coordinate_categorical_numeric_params() -> None: # Test with categorical params that can be interpreted as numeric params. study_categorical_params = create_study() study_categorical_params.add_trial( create_trial( value=0.0, params={ "category_a": 2, "category_b": 20 }, distributions={ "category_a": CategoricalDistribution((1, 2)), "category_b": CategoricalDistribution((10, 20, 30)), }, )) study_categorical_params.add_trial( create_trial( value=1.0, params={ "category_a": 1, "category_b": 30 }, distributions={ "category_a": CategoricalDistribution((1, 2)), "category_b": CategoricalDistribution((10, 20, 30)), }, )) study_categorical_params.add_trial( create_trial( value=2.0, params={ "category_a": 2, "category_b": 10 }, distributions={ "category_a": CategoricalDistribution((1, 2)), "category_b": CategoricalDistribution((10, 20, 30)), }, )) # Trials are sorted by using param_a and param_b, i.e., trial#1, trial#2, and trial#0. figure = plot_parallel_coordinate(study_categorical_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (1.0, 2.0, 0.0) assert figure.data[0]["dimensions"][1]["label"] == "category_a" assert figure.data[0]["dimensions"][1]["range"] == (0, 1) assert figure.data[0]["dimensions"][1]["values"] == (0, 1, 1) assert figure.data[0]["dimensions"][1]["ticktext"] == (1, 2) assert figure.data[0]["dimensions"][1]["tickvals"] == (0, 1) assert figure.data[0]["dimensions"][2]["label"] == "category_b" assert figure.data[0]["dimensions"][2]["range"] == (0, 2) assert figure.data[0]["dimensions"][2]["values"] == (2, 0, 1) assert figure.data[0]["dimensions"][2]["ticktext"] == (10, 20, 30) assert figure.data[0]["dimensions"][2]["tickvals"] == (0, 1, 2)
def log_study_info(study, experiment=None, log_charts=True, params=None): """Logs runs results and parameters to neptune. Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric), best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts ('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder. Args: study('optuna.study.Study'): Optuna study object after training is completed. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. log_charts('bool'): Whether optuna visualization charts should be logged. By default all charts are logged. params(`list`): List of parameters to be visualized. Default is all parameters. Examples: Initialize neptune_monitor:: import neptune import neptunecontrib.monitoring.optuna as opt_utils neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') neptune.create_experiment(name='optuna sweep') neptune_callback = opt_utils.NeptuneCallback() Run Optuna training passing monitor as callback:: ... study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=100, callbacks=[neptune_callback]) opt_utils.log_study_info(study) You can explore an example experiment in Neptune: https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts """ import optuna.visualization as vis _exp = experiment if experiment else neptune _exp.log_metric('best_score', study.best_value) _exp.set_property('best_parameters', study.best_params) if log_charts: log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp) log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp) log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp) log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp) pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
def test_color_map(direction: str) -> None: study = create_study(direction=direction) for i in range(3): study.add_trial( create_trial( value=float(i), params={"param_a": float(i), "param_b": float(i)}, distributions={ "param_a": FloatDistribution(0.0, 3.0), "param_b": FloatDistribution(0.0, 3.0), }, ) ) # `target` is `None`. line = plot_parallel_coordinate(study).data[0]["line"] assert COLOR_SCALE == [v[1] for v in line["colorscale"]] if direction == "minimize": assert line["reversescale"] else: assert not line["reversescale"] # When `target` is not `None`, `reversescale` is always `True`. line = plot_parallel_coordinate(study, target=lambda t: t.number).data[0]["line"] assert COLOR_SCALE == [v[1] for v in line["colorscale"]] assert line["reversescale"] # Multi-objective optimization. study = create_study(directions=[direction, direction]) for i in range(3): study.add_trial( create_trial( values=[float(i), float(i)], params={"param_a": float(i), "param_b": float(i)}, distributions={ "param_a": FloatDistribution(0.0, 3.0), "param_b": FloatDistribution(0.0, 3.0), }, ) ) line = plot_parallel_coordinate(study, target=lambda t: t.number).data[0]["line"] assert COLOR_SCALE == [v[1] for v in line["colorscale"]] assert line["reversescale"]
def draw_results(study): # 优化历史 plt.figure() fig = pv.plot_optimization_history(study) fig.write_image("./output/opt_his.png") plt.close() # 等高线图 plt.figure() fig = pv.plot_contour(study) fig.write_image("./output/opt_contour.png") plt.close() # 经验分布图 plt.figure() fig = pv.plot_edf(study) fig.write_image("./output/opt_edf.png") plt.close() # 高维参数 plt.figure() fig = pv.plot_parallel_coordinate(study) fig.write_image("./output/opt_coordinate.png") plt.close()
def test_plot_parallel_coordinate() -> None: # Test with no trial. study = create_study() figure = plot_parallel_coordinate(study) assert len(figure.data) == 0 study = create_study(direction="minimize") study.add_trial( create_trial( value=0.0, params={"param_a": 1.0, "param_b": 2.0}, distributions={ "param_a": FloatDistribution(0.0, 3.0), "param_b": FloatDistribution(0.0, 3.0), }, ) ) study.add_trial( create_trial( value=2.0, params={"param_b": 0.0}, distributions={"param_b": FloatDistribution(0.0, 3.0)}, ) ) study.add_trial( create_trial( value=1.0, params={"param_a": 2.5, "param_b": 1.0}, distributions={ "param_a": FloatDistribution(0.0, 3.0), "param_b": FloatDistribution(0.0, 3.0), }, ) ) # Test with a trial. figure = plot_parallel_coordinate(study) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0) assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0) # Test with a trial to select parameter. figure = plot_parallel_coordinate(study, params=["param_a"]) assert len(figure.data[0]["dimensions"]) == 2 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) # Test with a customized target value. with pytest.warns(UserWarning): figure = plot_parallel_coordinate( study, params=["param_a"], target=lambda t: t.params["param_b"] ) assert len(figure.data[0]["dimensions"]) == 2 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (1.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (2.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) # Test with a customized target name. figure = plot_parallel_coordinate(study, target_name="Target Name") assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Target Name" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0) assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0) # Test with wrong params that do not exist in trials with pytest.raises(ValueError, match="Parameter optuna does not exist in your study."): plot_parallel_coordinate(study, params=["optuna", "optuna"]) # Ignore failed trials. study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError,)) figure = plot_parallel_coordinate(study) assert len(figure.data) == 0
def test_target_is_none_and_study_is_multi_obj() -> None: study = create_study(directions=["minimize", "minimize"]) with pytest.raises(ValueError): plot_parallel_coordinate(study)
def test_plot_parallel_coordinate_with_categorical_numeric_params() -> None: # Test with sample from multiple distributions including categorical params # that can be interpreted as numeric params. study_multi_distro_params = create_study() study_multi_distro_params.add_trial( create_trial( value=0.0, params={ "param_a": "preferred", "param_b": 2, "param_c": 30, "param_d": 2 }, distributions={ "param_a": CategoricalDistribution(("preferred", "opt")), "param_b": CategoricalDistribution((1, 2, 10)), "param_c": FloatDistribution(1, 1000, log=True), "param_d": CategoricalDistribution((1, -1, 2)), }, )) study_multi_distro_params.add_trial( create_trial( value=1.0, params={ "param_a": "opt", "param_b": 1, "param_c": 200, "param_d": 2 }, distributions={ "param_a": CategoricalDistribution(("preferred", "opt")), "param_b": CategoricalDistribution((1, 2, 10)), "param_c": FloatDistribution(1, 1000, log=True), "param_d": CategoricalDistribution((1, -1, 2)), }, )) study_multi_distro_params.add_trial( create_trial( value=2.0, params={ "param_a": "preferred", "param_b": 10, "param_c": 10, "param_d": 1 }, distributions={ "param_a": CategoricalDistribution(("preferred", "opt")), "param_b": CategoricalDistribution((1, 2, 10)), "param_c": FloatDistribution(1, 1000, log=True), "param_d": CategoricalDistribution((1, -1, 2)), }, )) study_multi_distro_params.add_trial( create_trial( value=3.0, params={ "param_a": "opt", "param_b": 2, "param_c": 10, "param_d": -1 }, distributions={ "param_a": CategoricalDistribution(("preferred", "opt")), "param_b": CategoricalDistribution((1, 2, 10)), "param_c": FloatDistribution(1, 1000, log=True), "param_d": CategoricalDistribution((-1, 1, 2)), }, )) figure = plot_parallel_coordinate(study_multi_distro_params) assert len(figure.data[0]["dimensions"]) == 5 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 3.0) assert figure.data[0]["dimensions"][0]["values"] == (1.0, 3.0, 0.0, 2.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (0, 1) assert figure.data[0]["dimensions"][1]["values"] == (1, 1, 0, 0) assert figure.data[0]["dimensions"][1]["ticktext"] == ("preferred", "opt") assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (0, 2) assert figure.data[0]["dimensions"][2]["values"] == (0, 1, 1, 2) assert figure.data[0]["dimensions"][2]["ticktext"] == (1, 2, 10) assert figure.data[0]["dimensions"][3]["label"] == "param_c" assert figure.data[0]["dimensions"][3]["range"] == (1.0, math.log10(200)) assert figure.data[0]["dimensions"][3]["values"] == (math.log10(200), 1.0, math.log10(30), 1.0) assert figure.data[0]["dimensions"][3]["ticktext"] == ("10", "100", "200") assert figure.data[0]["dimensions"][3]["tickvals"] == (1.0, 2.0, math.log10(200)) assert figure.data[0]["dimensions"][4]["label"] == "param_d" assert figure.data[0]["dimensions"][4]["range"] == (0, 2) assert figure.data[0]["dimensions"][4]["values"] == (2, 0, 2, 1) assert figure.data[0]["dimensions"][4]["ticktext"] == (-1, 1, 2)
def test_nonfinite_removed(value: float) -> None: study = prepare_study_with_trials(value_for_first_trial=value) figure = plot_parallel_coordinate(study) assert all(np.isfinite(figure.data[0]["dimensions"][0]["values"]))
) study.optimize(objective, n_trials=100, timeout=600) ################################################################################################### # Plot functions # -------------- # Visualize the optimization history. See :func:`~optuna.visualization.plot_optimization_history` for the details. plot_optimization_history(study) ################################################################################################### # Visualize the learning curves of the trials. See :func:`~optuna.visualization.plot_intermediate_values` for the details. plot_intermediate_values(study) ################################################################################################### # Visualize high-dimensional parameter relationships. See :func:`~optuna.visualization.plot_parallel_coordinate` for the details. plot_parallel_coordinate(study) ################################################################################################### # Select parameters to visualize. plot_parallel_coordinate(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize hyperparameter relationships. See :func:`~optuna.visualization.plot_contour` for the details. plot_contour(study) ################################################################################################### # Select parameters to visualize. plot_contour(study, params=["bagging_freq", "bagging_fraction"]) ################################################################################################### # Visualize individual hyperparameters as slice plot. See :func:`~optuna.visualization.plot_slice` for the details.
def log_study_info(study, experiment=None, log_study=True, log_charts=True, log_optimization_history=False, log_contour=False, log_parallel_coordinate=False, log_slice=False, params=None): """Logs runs results and parameters to neptune. Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric), best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts ('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder. Args: study('optuna.study.Study'): Optuna study object after training is completed. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. log_study('bool'): Whether optuna study object should be logged as pickle. Default is True. log_charts('bool'): Deprecated argument. Whether all optuna visualizations charts should be logged. By default all charts are sent. To not log any charts set log_charts=False. If you want to log a particular chart change the argument for that chart explicitly. For example log_charts=False and log_slice=True will log only the slice plot to Neptune. log_optimization_history('bool'): Whether optuna optimization history chart should be logged. Default is True. log_contour('bool'): Whether optuna contour plot should be logged. Default is True. log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. Default is True. log_slice('bool'): Whether optuna slice chart should be logged. Default is True. params(`list`): List of parameters to be visualized. Default is all parameters. Examples: Initialize neptune_monitor:: import neptune import neptunecontrib.monitoring.optuna as opt_utils neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') neptune.create_experiment(name='optuna sweep') neptune_callback = opt_utils.NeptuneCallback() Run Optuna training passing monitor as callback:: ... study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=100, callbacks=[neptune_callback]) opt_utils.log_study_info(study) You can explore an example experiment in Neptune: https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts """ import optuna.visualization as vis _exp = experiment if experiment else neptune _exp.log_metric('best_score', study.best_value) _exp.set_property('best_parameters', study.best_params) if log_charts: message = """log_charts argument is depraceted and will be removed in future releases. Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly. """ warnings.warn(message) log_optimization_history = True log_contour = True log_parallel_coordinate = True log_slice = True if log_study: pickle_and_log_artifact(study, 'study.pkl', experiment=_exp) if log_optimization_history: log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp) if log_contour: log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp) if log_parallel_coordinate: log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp) if log_slice: log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp)
def ml_mlp_mul_ms(station_name="종로구"): print("Start Multivariate MLP Mean Seasonality Decomposition (MSE) Model") targets = ["PM10", "PM25"] # targets = ["SO2", "CO", "O3", "NO2", "PM10", "PM25", # "temp", "u", "v", "pres", "humid", "prep", "snow"] # 24*14 = 336 #sample_size = 336 sample_size = 48 output_size = 24 # If you want to debug, fast_dev_run = True and n_trials should be small number fast_dev_run = False n_trials = 128 # fast_dev_run = True # n_trials = 1 # Hyper parameter epoch_size = 500 batch_size = 64 learning_rate = 1e-3 # Blocked Cross Validation # neglect small overlap between train_dates and valid_dates # 11y = ((2y, 0.5y), (2y, 0.5y), (2y, 0.5y), (2.5y, 1y)) train_dates = [(dt.datetime(2008, 1, 4, 1).astimezone(SEOULTZ), dt.datetime(2009, 12, 31, 23).astimezone(SEOULTZ)), (dt.datetime(2010, 7, 1, 0).astimezone(SEOULTZ), dt.datetime(2012, 6, 30, 23).astimezone(SEOULTZ)), (dt.datetime(2013, 1, 1, 0).astimezone(SEOULTZ), dt.datetime(2014, 12, 31, 23).astimezone(SEOULTZ)), (dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ), dt.datetime(2017, 12, 31, 23).astimezone(SEOULTZ))] valid_dates = [(dt.datetime(2010, 1, 1, 0).astimezone(SEOULTZ), dt.datetime(2010, 6, 30, 23).astimezone(SEOULTZ)), (dt.datetime(2012, 7, 1, 0).astimezone(SEOULTZ), dt.datetime(2012, 12, 31, 23).astimezone(SEOULTZ)), (dt.datetime(2015, 1, 1, 0).astimezone(SEOULTZ), dt.datetime(2015, 6, 30, 23).astimezone(SEOULTZ)), (dt.datetime(2018, 1, 1, 0).astimezone(SEOULTZ), dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ))] train_valid_fdate = dt.datetime(2008, 1, 3, 1).astimezone(SEOULTZ) train_valid_tdate = dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ) # Debug if fast_dev_run: train_dates = [(dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ), dt.datetime(2017, 12, 31, 23).astimezone(SEOULTZ))] valid_dates = [(dt.datetime(2018, 1, 1, 0).astimezone(SEOULTZ), dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ))] train_valid_fdate = dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ) train_valid_tdate = dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ) test_fdate = dt.datetime(2019, 1, 1, 0).astimezone(SEOULTZ) test_tdate = dt.datetime(2020, 10, 31, 23).astimezone(SEOULTZ) # check date range assumption assert len(train_dates) == len(valid_dates) for i, (td, vd) in enumerate(zip(train_dates, valid_dates)): assert vd[0] > td[1] assert test_fdate > train_dates[-1][1] assert test_fdate > valid_dates[-1][1] train_features = [ "SO2", "CO", "NO2", "PM10", "PM25", "temp", "wind_spd", "wind_cdir", "wind_sdir", "pres", "humid", "prep" ] train_features_periodic = [ "SO2", "CO", "NO2", "PM10", "PM25", "temp", "wind_spd", "wind_cdir", "wind_sdir", "pres", "humid" ] train_features_nonperiodic = ["prep"] for target in targets: print("Training " + target + "...") output_dir = Path( f"/mnt/data/MLPMSMultivariate/{station_name}/{target}/") Path.mkdir(output_dir, parents=True, exist_ok=True) model_dir = output_dir / "models" Path.mkdir(model_dir, parents=True, exist_ok=True) log_dir = output_dir / "log" Path.mkdir(log_dir, parents=True, exist_ok=True) _df_h = data.load_imputed(HOURLY_DATA_PATH) df_h = _df_h.query('stationCode == "' + str(SEOUL_STATIONS[station_name]) + '"') if station_name == '종로구' and \ not Path("/input/python/input_jongno_imputed_hourly_pandas.csv").is_file(): # load imputed result df_h.to_csv("/input/python/input_jongno_imputed_hourly_pandas.csv") # construct dataset for seasonality print("Construct Train/Validation Sets...", flush=True) train_valid_dataset = construct_dataset(train_valid_fdate, train_valid_tdate, filepath=HOURLY_DATA_PATH, station_name=station_name, target=target, sample_size=sample_size, output_size=output_size, transform=False) # compute seasonality train_valid_dataset.preprocess() # For Block Cross Validation.. # load dataset in given range dates and transform using scaler from train_valid_set # all dataset are saved in tuple print("Construct Training Sets...", flush=True) train_datasets = tuple( construct_dataset(td[0], td[1], scaler_X=train_valid_dataset.scaler_X, scaler_Y=train_valid_dataset.scaler_Y, filepath=HOURLY_DATA_PATH, station_name=station_name, target=target, sample_size=sample_size, output_size=output_size, features=train_features, features_periodic=train_features_periodic, features_nonperiodic=train_features_nonperiodic, transform=True) for td in train_dates) print("Construct Validation Sets...", flush=True) valid_datasets = tuple( construct_dataset(vd[0], vd[1], scaler_X=train_valid_dataset.scaler_X, scaler_Y=train_valid_dataset.scaler_Y, filepath=HOURLY_DATA_PATH, station_name=station_name, target=target, sample_size=sample_size, output_size=output_size, features=train_features, features_periodic=train_features_periodic, features_nonperiodic=train_features_nonperiodic, transform=True) for vd in valid_dates) # just single test set print("Construct Test Sets...", flush=True) test_dataset = construct_dataset( test_fdate, test_tdate, scaler_X=train_valid_dataset.scaler_X, scaler_Y=train_valid_dataset.scaler_Y, filepath=HOURLY_DATA_PATH, station_name=station_name, target=target, sample_size=sample_size, output_size=output_size, features=train_features, features_periodic=train_features_periodic, features_nonperiodic=train_features_nonperiodic, transform=True) # convert tuple of datasets to ConcatDataset train_dataset = ConcatDataset(train_datasets) val_dataset = ConcatDataset(valid_datasets) # num_layer == number of hidden layer hparams = Namespace(num_layers=1, layer_size=128, learning_rate=learning_rate, batch_size=batch_size) def objective(trial): model = BaseMLPModel( trial=trial, hparams=hparams, input_size=sample_size * len(train_features), sample_size=sample_size, output_size=output_size, station_name=station_name, target=target, features=train_features, features_periodic=train_features_periodic, features_nonperiodic=train_features_nonperiodic, train_dataset=train_dataset, val_dataset=val_dataset, test_dataset=test_dataset, scaler_X=train_valid_dataset.scaler_X, scaler_Y=train_valid_dataset.scaler_Y, output_dir=output_dir) # most basic trainer, uses good defaults trainer = Trainer(gpus=1 if torch.cuda.is_available() else None, precision=32, min_epochs=1, max_epochs=20, default_root_dir=output_dir, fast_dev_run=fast_dev_run, logger=True, checkpoint_callback=False, callbacks=[ PyTorchLightningPruningCallback( trial, monitor="valid/MSE") ]) trainer.fit(model) # Don't Log # hyperparameters = model.hparams # trainer.logger.log_hyperparams(hyperparameters) return trainer.callback_metrics.get("valid/MSE") if n_trials > 1: study = optuna.create_study(direction="minimize") study.enqueue_trial({ 'sigma': 1.3, 'num_layers': 4, 'layer_size': 8, 'learning_rate': learning_rate, 'batch_size': batch_size }) study.enqueue_trial({ 'sigma': 1.3, 'num_layers': 4, 'layer_size': 32, 'learning_rate': learning_rate, 'batch_size': batch_size }) study.enqueue_trial({ 'sigma': 1.3, 'num_layers': 4, 'layer_size': 64, 'learning_rate': learning_rate, 'batch_size': batch_size }) study.enqueue_trial({ 'sigma': 1.3, 'num_layers': 4, 'layer_size': 32, 'learning_rate': learning_rate, 'batch_size': batch_size }) study.enqueue_trial({ 'sigma': 1.3, 'num_layers': 8, 'layer_size': 32, 'learning_rate': learning_rate, 'batch_size': batch_size }) study.enqueue_trial({ 'sigma': 1.3, 'num_layers': 12, 'layer_size': 32, 'learning_rate': learning_rate, 'batch_size': batch_size }) study.enqueue_trial({ 'sigma': 0.7, 'num_layers': 4, 'layer_size': 32, 'learning_rate': learning_rate, 'batch_size': batch_size }) study.enqueue_trial({ 'sigma': 2.0, 'num_layers': 4, 'layer_size': 32, 'learning_rate': learning_rate, 'batch_size': batch_size }) # timeout = 3600*36 = 36h study.optimize(objective, n_trials=n_trials, timeout=3600 * 36) trial = study.best_trial print(" Value: ", trial.value) print(" Params: ") for key, value in trial.params.items(): print(" {}: {}".format(key, value)) print("sample_size : ", sample_size) print("output_size : ", output_size) # plot optmization results fig_cont1 = optv.plot_contour(study, params=['num_layers', 'layer_size']) fig_cont1.write_image( str(output_dir / "contour_num_layers_layer_size.png")) fig_cont1.write_image( str(output_dir / "contour_num_layers_layer_size.svg")) fig_edf = optv.plot_edf(study) fig_edf.write_image(str(output_dir / "edf.png")) fig_edf.write_image(str(output_dir / "edf.svg")) fig_iv = optv.plot_intermediate_values(study) fig_iv.write_image(str(output_dir / "intermediate_values.png")) fig_iv.write_image(str(output_dir / "intermediate_values.svg")) fig_his = optv.plot_optimization_history(study) fig_his.write_image(str(output_dir / "opt_history.png")) fig_his.write_image(str(output_dir / "opt_history.svg")) fig_pcoord = optv.plot_parallel_coordinate( study, params=['num_layers', 'layer_size']) fig_pcoord.write_image(str(output_dir / "parallel_coord.png")) fig_pcoord.write_image(str(output_dir / "parallel_coord.svg")) fig_slice = optv.plot_slice(study, params=['num_layers', 'layer_size']) fig_slice.write_image(str(output_dir / "slice.png")) fig_slice.write_image(str(output_dir / "slice.svg")) # set hparams with optmized value hparams.num_layers = trial.params['num_layers'] hparams.layer_size = trial.params['layer_size'] dict_hparams = copy.copy(vars(hparams)) dict_hparams["sample_size"] = sample_size dict_hparams["output_size"] = output_size with open(output_dir / 'hparams.json', 'w') as f: print(dict_hparams, file=f) with open(output_dir / 'hparams.csv', 'w') as f: print(pd.DataFrame.from_dict(dict_hparams, orient='index'), file=f) model = BaseMLPModel(hparams=hparams, input_size=sample_size * len(train_features), sample_size=sample_size, output_size=output_size, station_name=station_name, target=target, features=train_features, features_periodic=train_features_periodic, features_nonperiodic=train_features_nonperiodic, train_dataset=train_dataset, val_dataset=val_dataset, test_dataset=test_dataset, scaler_X=train_valid_dataset.scaler_X, scaler_Y=train_valid_dataset.scaler_Y, output_dir=output_dir) # record input for i, _train_set in enumerate(train_datasets): _train_set.to_csv( model.data_dir / ("df_trainset_{0}_".format(str(i).zfill(2)) + target + ".csv")) for i, _valid_set in enumerate(valid_datasets): _valid_set.to_csv( model.data_dir / ("df_validset_{0}_".format(str(i).zfill(2)) + target + ".csv")) train_valid_dataset.to_csv(model.data_dir / ("df_trainvalidset_" + target + ".csv")) test_dataset.to_csv(model.data_dir / ("df_testset_" + target + ".csv")) checkpoint_callback = pl.callbacks.ModelCheckpoint(os.path.join( model_dir, "train_{epoch}_{valid/MSE:.2f}"), monitor="valid/MSE", period=10) early_stop_callback = EarlyStopping(monitor='valid/MSE', min_delta=0.001, patience=30, verbose=True, mode='min') log_version = dt.date.today().strftime("%y%m%d-%H-%M") loggers = [ \ TensorBoardLogger(log_dir, version=log_version), CSVLogger(log_dir, version=log_version)] # most basic trainer, uses good defaults trainer = Trainer(gpus=1 if torch.cuda.is_available() else None, precision=32, min_epochs=1, max_epochs=epoch_size, default_root_dir=output_dir, fast_dev_run=fast_dev_run, logger=loggers, log_every_n_steps=5, flush_logs_every_n_steps=10, callbacks=[early_stop_callback], checkpoint_callback=checkpoint_callback) trainer.fit(model) # run test set trainer.test(ckpt_path=None) shutil.rmtree(model_dir)
def show_parallel_coordinates(self, params=None): return visualization.plot_parallel_coordinate(self.study, params=params)
layerin = Input(name='input', shape=(X.shape[-1],)) hidden = Dense(trial.suggest_int('n_hidden_units_enc1', 10, 64, 8), activation=act)(layerin) hidden = Dense(trial.suggest_int(f'n_hidden_units_enc2', 4, 64, 8), activation=act)(hidden) dense = Dense(name='encoded', units=2, activation='linear')(hidden) # tanh oder linear or relu or elu all work ok-ish layerout = Dense(units=X.shape[-1])(dense) # (hidden_decode) model = Model(layerin, layerout) model.compile(trial.suggest_categorical('optim', ['adam', 'rmsprop', 'sgd']), 'mean_squared_error') model.fit(x=X, y=X, epochs=150, batch_size=32, callbacks=[EarlyStopping(patience=5, monitor='loss'), TFKerasPruningCallback(trial, 'loss')]) return MeanSquaredError()(X, model.predict(X)) study = optuna.create_study(direction='minimize') study.optimize(objective, n_trials=10) #%% study.trials_dataframe().sort_values('value') # %% from optuna.visualization import plot_parallel_coordinate plot_parallel_coordinate(study) #%% from optuna.importance import get_param_importances get_param_importances(study) #%% import hiplot as hip hip.Experiment.from_dataframe(study.trials_dataframe()).display()
if __name__ == "__main__": study = optuna.create_study(direction="maximize", pruner=optuna.pruners.MedianPruner()) study.optimize(objective, n_trials=100, timeout=600) # Visualize the optimization history. plot_optimization_history(study).show() # Visualize the learning curves of the trials. plot_intermediate_values(study).show() # Visualize high-dimensional parameter relationships. plot_parallel_coordinate(study).show() # Select parameters to visualize. plot_parallel_coordinate(study, params=["lr_init", "n_units_l0"]).show() # Visualize hyperparameter relationships. plot_contour(study).show() # Select parameters to visualize. plot_contour(study, params=["n_units_l0", "n_units_l1"]).show() # Visualize individual hyperparameters. plot_slice(study).show() # Select parameters to visualize. plot_slice(study, params=["n_units_l0", "n_units_l1"]).show()
def test_plot_parallel_coordinate() -> None: # Test with no trial. study = create_study() figure = plot_parallel_coordinate(study) assert len(figure.data) == 0 study = prepare_study_with_trials(with_c_d=False) # Test with a trial. figure = plot_parallel_coordinate(study) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0) assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0) # Test with a trial to select parameter. figure = plot_parallel_coordinate(study, params=["param_a"]) assert len(figure.data[0]["dimensions"]) == 2 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) # Test with a customized target value. with pytest.warns(UserWarning): figure = plot_parallel_coordinate(study, params=["param_a"], target=lambda t: t.params["param_b"]) assert len(figure.data[0]["dimensions"]) == 2 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (1.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (2.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) # Test with a customized target name. figure = plot_parallel_coordinate(study, target_name="Target Name") assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Target Name" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 1.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (1.0, 2.0) assert figure.data[0]["dimensions"][2]["values"] == (2.0, 1.0) # Test with wrong params that do not exist in trials with pytest.raises(ValueError, match="Parameter optuna does not exist in your study."): plot_parallel_coordinate(study, params=["optuna", "optuna"]) # Ignore failed trials. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError, )) figure = plot_parallel_coordinate(study) assert len(figure.data) == 0
def test_plot_parallel_coordinate() -> None: # Test with no trial. study = create_study() figure = plot_parallel_coordinate(study) assert len(figure.data) == 0 study = prepare_study_with_trials(with_c_d=False) # Test with a trial. figure = plot_parallel_coordinate(study) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][2]["values"] == (2.0, 0.0, 1.0) # Test with a trial to select parameter. figure = plot_parallel_coordinate(study, params=["param_a"]) assert len(figure.data[0]["dimensions"]) == 2 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) # Test with a customized target value. figure = plot_parallel_coordinate(study, params=["param_a"], target=lambda t: t.params["param_b"]) assert len(figure.data[0]["dimensions"]) == 2 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (2.0, 0.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) # Test with a customized target name. figure = plot_parallel_coordinate(study, target_name="Target Name") assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Target Name" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0, 1.0) assert figure.data[0]["dimensions"][1]["label"] == "param_a" assert figure.data[0]["dimensions"][1]["range"] == (1.0, 2.5) assert figure.data[0]["dimensions"][1]["values"] == (1.0, 2.5) assert figure.data[0]["dimensions"][2]["label"] == "param_b" assert figure.data[0]["dimensions"][2]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][2]["values"] == (2.0, 0.0, 1.0) # Test with wrong params that do not exist in trials with pytest.raises(ValueError, match="Parameter optuna does not exist in your study."): plot_parallel_coordinate(study, params=["optuna", "optuna"]) # Ignore failed trials. def fail_objective(_: Trial) -> float: raise ValueError study = create_study() study.optimize(fail_objective, n_trials=1, catch=(ValueError, )) figure = plot_parallel_coordinate(study) assert len(figure.data) == 0 # Test with categorical params that cannot be converted to numeral. study_categorical_params = create_study() study_categorical_params.add_trial( create_trial( value=0.0, params={ "category_a": "preferred", "category_b": "net" }, distributions={ "category_a": CategoricalDistribution(("preferred", "opt")), "category_b": CategoricalDistribution(("net", "una")), }, )) study_categorical_params.add_trial( create_trial( value=2.0, params={ "category_a": "opt", "category_b": "una" }, distributions={ "category_a": CategoricalDistribution(("preferred", "opt")), "category_b": CategoricalDistribution(("net", "una")), }, )) figure = plot_parallel_coordinate(study_categorical_params) assert len(figure.data[0]["dimensions"]) == 3 assert figure.data[0]["dimensions"][0]["label"] == "Objective Value" assert figure.data[0]["dimensions"][0]["range"] == (0.0, 2.0) assert figure.data[0]["dimensions"][0]["values"] == (0.0, 2.0) assert figure.data[0]["dimensions"][1]["label"] == "category_a" assert figure.data[0]["dimensions"][1]["range"] == (0, 1) assert figure.data[0]["dimensions"][1]["values"] == (0, 1) assert figure.data[0]["dimensions"][1]["ticktext"] == (["preferred", 0], ["opt", 1]) assert figure.data[0]["dimensions"][2]["label"] == "category_b" assert figure.data[0]["dimensions"][2]["range"] == (0, 1) assert figure.data[0]["dimensions"][2]["values"] == (0, 1) assert figure.data[0]["dimensions"][2]["ticktext"] == (["net", 0], ["una", 1])
def test_nonfinite_multiobjective(objective: int, value: float) -> None: study = prepare_study_with_trials(n_objectives=2, value_for_first_trial=value) figure = plot_parallel_coordinate(study, target=lambda t: t.values[objective]) assert all(np.isfinite(figure.data[0]["dimensions"][0]["values"]))