Example #1
0
def make_plots(logdir, study):
    logdir = f'{logdir}/plots'
    os.makedirs(logdir, exist_ok=True)
    plot_optimization_history(study).write_image(f'{logdir}/history.svg')
    plot_intermediate_values(study).write_image(f'{logdir}/intermediates.svg')
    plot_parallel_coordinate(study).write_image(f'{logdir}/parallel_coordinates.png')
    plot_slice(study).write_image(f'{logdir}/slices.svg')
    plot_param_importances(study).write_image(f'{logdir}/importances.svg')
Example #2
0
def test_plot_intermediate_values():
    # type: () -> None

    # Test with no trials.
    study = prepare_study_with_trials(no_trials=True)
    figure = plot_intermediate_values(study)
    assert not figure.data

    def objective(trial, report_intermediate_values):
        # type: (Trial, bool) -> float

        if report_intermediate_values:
            trial.report(1.0, step=0)
            trial.report(2.0, step=1)
        return 0.0

    # Test with a trial with intermediate values.
    study = create_study()
    study.optimize(lambda t: objective(t, True), n_trials=1)
    figure = plot_intermediate_values(study)
    assert len(figure.data) == 1
    assert figure.data[0].x == (0, 1)
    assert figure.data[0].y == (1.0, 2.0)

    # Test a study with one trial with intermediate values and
    # one trial without intermediate values.
    # Expect the trial with no intermediate values to be ignored.
    study.optimize(lambda t: objective(t, False), n_trials=1)
    assert len(study.trials) == 2
    figure = plot_intermediate_values(study)
    assert len(figure.data) == 1
    assert figure.data[0].x == (0, 1)
    assert figure.data[0].y == (1.0, 2.0)

    # Test a study of only one trial that has no intermediate values.
    study = create_study()
    study.optimize(lambda t: objective(t, False), n_trials=1)
    figure = plot_intermediate_values(study)
    assert not figure.data

    # Ignore failed trials.
    def fail_objective(_):
        # type: (Trial) -> float

        raise ValueError

    study = create_study()
    study.optimize(fail_objective, n_trials=1, catch=(ValueError,))
    figure = plot_intermediate_values(study)
    assert not figure.data
Example #3
0
    def plot_intermediate_values(self, interactive=False, legend=False):
        '''
        Plot optimization trials history. Shows successful and terminated trials.

        Parameters
        ----------
        interactive : bool, optional
            Create & show in default browsersave to current wd interactive html plot. The default is False.
        legend : bool, optional
            Flag to include legend in the static (not interactive) plot. The default is False.

        Returns
        -------
        None.

        '''
        self._check_refit_status('plot_intermediate_values()')
        validate_plotting_interactive_argument(interactive)
        validate_plotting_legend_argument(legend)

        if interactive:
            from optuna.visualization import plot_intermediate_values
            fig = plot_intermediate_values(self._study)
            fig.write_html("intermediate_values_plot.html")
            try:
                self._display_html('intermediate_values_plot.html')
            except Exception as e:
                print(f'Display html error: {e}')
                print(
                    f'Intermediate Values Plot is saved to {os.path.join(os.getcwd(), "intermediate_values_plot.html")}'
                )
        else:
            from optuna.visualization.matplotlib import plot_intermediate_values
            import matplotlib.pyplot as plt
            fig = plot_intermediate_values(self._study)
            if not legend:
                fig.get_legend().remove()
            plt.show()
study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.TPESampler(seed=SEED),
    pruner=optuna.pruners.MedianPruner(n_warmup_steps=10),
)
study.optimize(objective, n_trials=100, timeout=600)

###################################################################################################
# Plot functions
# --------------
# Visualize the optimization history. See :func:`~optuna.visualization.plot_optimization_history` for the details.
plot_optimization_history(study)

###################################################################################################
# Visualize the learning curves of the trials. See :func:`~optuna.visualization.plot_intermediate_values` for the details.
plot_intermediate_values(study)

###################################################################################################
# Visualize high-dimensional parameter relationships. See :func:`~optuna.visualization.plot_parallel_coordinate` for the details.
plot_parallel_coordinate(study)

###################################################################################################
# Select parameters to visualize.
plot_parallel_coordinate(study, params=["bagging_freq", "bagging_fraction"])

###################################################################################################
# Visualize hyperparameter relationships. See :func:`~optuna.visualization.plot_contour` for the details.
plot_contour(study)

###################################################################################################
# Select parameters to visualize.
Example #5
0
            raise optuna.TrialPruned()

    return value


if __name__ == "__main__":

    study = optuna.create_study(direction="maximize",
                                pruner=optuna.pruners.MedianPruner())
    study.optimize(objective, n_trials=100, timeout=600)

    # Visualize the optimization history.
    plot_optimization_history(study).show()

    # Visualize the learning curves of the trials.
    plot_intermediate_values(study).show()

    # Visualize high-dimensional parameter relationships.
    plot_parallel_coordinate(study).show()

    # Select parameters to visualize.
    plot_parallel_coordinate(study, params=["lr_init", "n_units_l0"]).show()

    # Visualize hyperparameter relationships.
    plot_contour(study).show()

    # Select parameters to visualize.
    plot_contour(study, params=["n_units_l0", "n_units_l1"]).show()

    # Visualize individual hyperparameters.
    plot_slice(study).show()
Example #6
0
def ml_mlp_mul_ms(station_name="종로구"):
    print("Start Multivariate MLP Mean Seasonality Decomposition (MSE) Model")
    targets = ["PM10", "PM25"]
    # targets = ["SO2", "CO", "O3", "NO2", "PM10", "PM25",
    #                   "temp", "u", "v", "pres", "humid", "prep", "snow"]
    # 24*14 = 336
    #sample_size = 336
    sample_size = 48
    output_size = 24
    # If you want to debug, fast_dev_run = True and n_trials should be small number
    fast_dev_run = False
    n_trials = 128
    # fast_dev_run = True
    # n_trials = 1

    # Hyper parameter
    epoch_size = 500
    batch_size = 64
    learning_rate = 1e-3

    # Blocked Cross Validation
    # neglect small overlap between train_dates and valid_dates
    # 11y = ((2y, 0.5y), (2y, 0.5y), (2y, 0.5y), (2.5y, 1y))
    train_dates = [(dt.datetime(2008, 1, 4, 1).astimezone(SEOULTZ),
                    dt.datetime(2009, 12, 31, 23).astimezone(SEOULTZ)),
                   (dt.datetime(2010, 7, 1, 0).astimezone(SEOULTZ),
                    dt.datetime(2012, 6, 30, 23).astimezone(SEOULTZ)),
                   (dt.datetime(2013, 1, 1, 0).astimezone(SEOULTZ),
                    dt.datetime(2014, 12, 31, 23).astimezone(SEOULTZ)),
                   (dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ),
                    dt.datetime(2017, 12, 31, 23).astimezone(SEOULTZ))]
    valid_dates = [(dt.datetime(2010, 1, 1, 0).astimezone(SEOULTZ),
                    dt.datetime(2010, 6, 30, 23).astimezone(SEOULTZ)),
                   (dt.datetime(2012, 7, 1, 0).astimezone(SEOULTZ),
                    dt.datetime(2012, 12, 31, 23).astimezone(SEOULTZ)),
                   (dt.datetime(2015, 1, 1, 0).astimezone(SEOULTZ),
                    dt.datetime(2015, 6, 30, 23).astimezone(SEOULTZ)),
                   (dt.datetime(2018, 1, 1, 0).astimezone(SEOULTZ),
                    dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ))]
    train_valid_fdate = dt.datetime(2008, 1, 3, 1).astimezone(SEOULTZ)
    train_valid_tdate = dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ)

    # Debug
    if fast_dev_run:
        train_dates = [(dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ),
                        dt.datetime(2017, 12, 31, 23).astimezone(SEOULTZ))]
        valid_dates = [(dt.datetime(2018, 1, 1, 0).astimezone(SEOULTZ),
                        dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ))]
        train_valid_fdate = dt.datetime(2015, 7, 1, 0).astimezone(SEOULTZ)
        train_valid_tdate = dt.datetime(2018, 12, 31, 23).astimezone(SEOULTZ)

    test_fdate = dt.datetime(2019, 1, 1, 0).astimezone(SEOULTZ)
    test_tdate = dt.datetime(2020, 10, 31, 23).astimezone(SEOULTZ)

    # check date range assumption
    assert len(train_dates) == len(valid_dates)
    for i, (td, vd) in enumerate(zip(train_dates, valid_dates)):
        assert vd[0] > td[1]
    assert test_fdate > train_dates[-1][1]
    assert test_fdate > valid_dates[-1][1]

    train_features = [
        "SO2", "CO", "NO2", "PM10", "PM25", "temp", "wind_spd", "wind_cdir",
        "wind_sdir", "pres", "humid", "prep"
    ]
    train_features_periodic = [
        "SO2", "CO", "NO2", "PM10", "PM25", "temp", "wind_spd", "wind_cdir",
        "wind_sdir", "pres", "humid"
    ]
    train_features_nonperiodic = ["prep"]

    for target in targets:
        print("Training " + target + "...")
        output_dir = Path(
            f"/mnt/data/MLPMSMultivariate/{station_name}/{target}/")
        Path.mkdir(output_dir, parents=True, exist_ok=True)
        model_dir = output_dir / "models"
        Path.mkdir(model_dir, parents=True, exist_ok=True)
        log_dir = output_dir / "log"
        Path.mkdir(log_dir, parents=True, exist_ok=True)

        _df_h = data.load_imputed(HOURLY_DATA_PATH)
        df_h = _df_h.query('stationCode == "' +
                           str(SEOUL_STATIONS[station_name]) + '"')

        if station_name == '종로구' and \
            not Path("/input/python/input_jongno_imputed_hourly_pandas.csv").is_file():
            # load imputed result

            df_h.to_csv("/input/python/input_jongno_imputed_hourly_pandas.csv")

        # construct dataset for seasonality
        print("Construct Train/Validation Sets...", flush=True)
        train_valid_dataset = construct_dataset(train_valid_fdate,
                                                train_valid_tdate,
                                                filepath=HOURLY_DATA_PATH,
                                                station_name=station_name,
                                                target=target,
                                                sample_size=sample_size,
                                                output_size=output_size,
                                                transform=False)
        # compute seasonality
        train_valid_dataset.preprocess()

        # For Block Cross Validation..
        # load dataset in given range dates and transform using scaler from train_valid_set
        # all dataset are saved in tuple
        print("Construct Training Sets...", flush=True)
        train_datasets = tuple(
            construct_dataset(td[0],
                              td[1],
                              scaler_X=train_valid_dataset.scaler_X,
                              scaler_Y=train_valid_dataset.scaler_Y,
                              filepath=HOURLY_DATA_PATH,
                              station_name=station_name,
                              target=target,
                              sample_size=sample_size,
                              output_size=output_size,
                              features=train_features,
                              features_periodic=train_features_periodic,
                              features_nonperiodic=train_features_nonperiodic,
                              transform=True) for td in train_dates)

        print("Construct Validation Sets...", flush=True)
        valid_datasets = tuple(
            construct_dataset(vd[0],
                              vd[1],
                              scaler_X=train_valid_dataset.scaler_X,
                              scaler_Y=train_valid_dataset.scaler_Y,
                              filepath=HOURLY_DATA_PATH,
                              station_name=station_name,
                              target=target,
                              sample_size=sample_size,
                              output_size=output_size,
                              features=train_features,
                              features_periodic=train_features_periodic,
                              features_nonperiodic=train_features_nonperiodic,
                              transform=True) for vd in valid_dates)

        # just single test set
        print("Construct Test Sets...", flush=True)
        test_dataset = construct_dataset(
            test_fdate,
            test_tdate,
            scaler_X=train_valid_dataset.scaler_X,
            scaler_Y=train_valid_dataset.scaler_Y,
            filepath=HOURLY_DATA_PATH,
            station_name=station_name,
            target=target,
            sample_size=sample_size,
            output_size=output_size,
            features=train_features,
            features_periodic=train_features_periodic,
            features_nonperiodic=train_features_nonperiodic,
            transform=True)

        # convert tuple of datasets to ConcatDataset
        train_dataset = ConcatDataset(train_datasets)
        val_dataset = ConcatDataset(valid_datasets)

        # num_layer == number of hidden layer
        hparams = Namespace(num_layers=1,
                            layer_size=128,
                            learning_rate=learning_rate,
                            batch_size=batch_size)

        def objective(trial):
            model = BaseMLPModel(
                trial=trial,
                hparams=hparams,
                input_size=sample_size * len(train_features),
                sample_size=sample_size,
                output_size=output_size,
                station_name=station_name,
                target=target,
                features=train_features,
                features_periodic=train_features_periodic,
                features_nonperiodic=train_features_nonperiodic,
                train_dataset=train_dataset,
                val_dataset=val_dataset,
                test_dataset=test_dataset,
                scaler_X=train_valid_dataset.scaler_X,
                scaler_Y=train_valid_dataset.scaler_Y,
                output_dir=output_dir)

            # most basic trainer, uses good defaults
            trainer = Trainer(gpus=1 if torch.cuda.is_available() else None,
                              precision=32,
                              min_epochs=1,
                              max_epochs=20,
                              default_root_dir=output_dir,
                              fast_dev_run=fast_dev_run,
                              logger=True,
                              checkpoint_callback=False,
                              callbacks=[
                                  PyTorchLightningPruningCallback(
                                      trial, monitor="valid/MSE")
                              ])

            trainer.fit(model)

            # Don't Log
            # hyperparameters = model.hparams
            # trainer.logger.log_hyperparams(hyperparameters)

            return trainer.callback_metrics.get("valid/MSE")

        if n_trials > 1:
            study = optuna.create_study(direction="minimize")
            study.enqueue_trial({
                'sigma': 1.3,
                'num_layers': 4,
                'layer_size': 8,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            study.enqueue_trial({
                'sigma': 1.3,
                'num_layers': 4,
                'layer_size': 32,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            study.enqueue_trial({
                'sigma': 1.3,
                'num_layers': 4,
                'layer_size': 64,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            study.enqueue_trial({
                'sigma': 1.3,
                'num_layers': 4,
                'layer_size': 32,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            study.enqueue_trial({
                'sigma': 1.3,
                'num_layers': 8,
                'layer_size': 32,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            study.enqueue_trial({
                'sigma': 1.3,
                'num_layers': 12,
                'layer_size': 32,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            study.enqueue_trial({
                'sigma': 0.7,
                'num_layers': 4,
                'layer_size': 32,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            study.enqueue_trial({
                'sigma': 2.0,
                'num_layers': 4,
                'layer_size': 32,
                'learning_rate': learning_rate,
                'batch_size': batch_size
            })
            # timeout = 3600*36 = 36h
            study.optimize(objective, n_trials=n_trials, timeout=3600 * 36)

            trial = study.best_trial

            print("  Value: ", trial.value)

            print("  Params: ")
            for key, value in trial.params.items():
                print("    {}: {}".format(key, value))
            print("sample_size : ", sample_size)
            print("output_size : ", output_size)

            # plot optmization results
            fig_cont1 = optv.plot_contour(study,
                                          params=['num_layers', 'layer_size'])
            fig_cont1.write_image(
                str(output_dir / "contour_num_layers_layer_size.png"))
            fig_cont1.write_image(
                str(output_dir / "contour_num_layers_layer_size.svg"))

            fig_edf = optv.plot_edf(study)
            fig_edf.write_image(str(output_dir / "edf.png"))
            fig_edf.write_image(str(output_dir / "edf.svg"))

            fig_iv = optv.plot_intermediate_values(study)
            fig_iv.write_image(str(output_dir / "intermediate_values.png"))
            fig_iv.write_image(str(output_dir / "intermediate_values.svg"))

            fig_his = optv.plot_optimization_history(study)
            fig_his.write_image(str(output_dir / "opt_history.png"))
            fig_his.write_image(str(output_dir / "opt_history.svg"))

            fig_pcoord = optv.plot_parallel_coordinate(
                study, params=['num_layers', 'layer_size'])
            fig_pcoord.write_image(str(output_dir / "parallel_coord.png"))
            fig_pcoord.write_image(str(output_dir / "parallel_coord.svg"))

            fig_slice = optv.plot_slice(study,
                                        params=['num_layers', 'layer_size'])
            fig_slice.write_image(str(output_dir / "slice.png"))
            fig_slice.write_image(str(output_dir / "slice.svg"))

            # set hparams with optmized value
            hparams.num_layers = trial.params['num_layers']
            hparams.layer_size = trial.params['layer_size']

            dict_hparams = copy.copy(vars(hparams))
            dict_hparams["sample_size"] = sample_size
            dict_hparams["output_size"] = output_size
            with open(output_dir / 'hparams.json', 'w') as f:
                print(dict_hparams, file=f)
            with open(output_dir / 'hparams.csv', 'w') as f:
                print(pd.DataFrame.from_dict(dict_hparams, orient='index'),
                      file=f)

        model = BaseMLPModel(hparams=hparams,
                             input_size=sample_size * len(train_features),
                             sample_size=sample_size,
                             output_size=output_size,
                             station_name=station_name,
                             target=target,
                             features=train_features,
                             features_periodic=train_features_periodic,
                             features_nonperiodic=train_features_nonperiodic,
                             train_dataset=train_dataset,
                             val_dataset=val_dataset,
                             test_dataset=test_dataset,
                             scaler_X=train_valid_dataset.scaler_X,
                             scaler_Y=train_valid_dataset.scaler_Y,
                             output_dir=output_dir)

        # record input
        for i, _train_set in enumerate(train_datasets):
            _train_set.to_csv(
                model.data_dir /
                ("df_trainset_{0}_".format(str(i).zfill(2)) + target + ".csv"))
        for i, _valid_set in enumerate(valid_datasets):
            _valid_set.to_csv(
                model.data_dir /
                ("df_validset_{0}_".format(str(i).zfill(2)) + target + ".csv"))
        train_valid_dataset.to_csv(model.data_dir /
                                   ("df_trainvalidset_" + target + ".csv"))
        test_dataset.to_csv(model.data_dir / ("df_testset_" + target + ".csv"))

        checkpoint_callback = pl.callbacks.ModelCheckpoint(os.path.join(
            model_dir, "train_{epoch}_{valid/MSE:.2f}"),
                                                           monitor="valid/MSE",
                                                           period=10)

        early_stop_callback = EarlyStopping(monitor='valid/MSE',
                                            min_delta=0.001,
                                            patience=30,
                                            verbose=True,
                                            mode='min')

        log_version = dt.date.today().strftime("%y%m%d-%H-%M")
        loggers = [ \
            TensorBoardLogger(log_dir, version=log_version),
            CSVLogger(log_dir, version=log_version)]

        # most basic trainer, uses good defaults
        trainer = Trainer(gpus=1 if torch.cuda.is_available() else None,
                          precision=32,
                          min_epochs=1,
                          max_epochs=epoch_size,
                          default_root_dir=output_dir,
                          fast_dev_run=fast_dev_run,
                          logger=loggers,
                          log_every_n_steps=5,
                          flush_logs_every_n_steps=10,
                          callbacks=[early_stop_callback],
                          checkpoint_callback=checkpoint_callback)

        trainer.fit(model)

        # run test set
        trainer.test(ckpt_path=None)

        shutil.rmtree(model_dir)