Ejemplo n.º 1
0
def main():
    """
    Get data from db and save it as csv
    """

    bq = BQHandler()
    io = IO(gs_bucket=options.gs_bucket)
    viz = Viz(io=io)

    starttime, endtime = io.get_dates(options)
    logging.info('Using dataset {} and time range {} - {}'.format(
        options.feature_dataset, starttime.strftime('%Y-%m-%d'),
        endtime.strftime('%Y-%m-%d')))

    all_param_names = options.label_params + options.feature_params + options.meta_params
    aggs = io.get_aggs_from_param_names(options.feature_params)

    if options.model == 'rf':
        model = RandomForestRegressor(
            n_estimators=options.n_estimators,
            n_jobs=-1,
            min_samples_leaf=options.min_samples_leaf,
            min_samples_split=options.min_samples_split,
            max_features=options.max_features,
            max_depth=options.max_depth,
            bootstrap=options.bootstrap)
    elif options.model == 'lr':
        model = SGDRegressor(warm_start=True,
                             max_iter=options.n_loops,
                             shuffle=options.shuffle,
                             power_t=options.power_t,
                             penalty=options.regularizer,
                             learning_rate=options.learning_rate,
                             eta0=options.eta0,
                             alpha=options.alpha,
                             tol=0.0001)
    elif options.model == 'svr':
        model = SVR()
    elif options.model == 'ard':
        model = ARDRegression(n_iter=options.n_loops,
                              alpha_1=options.alpha_1,
                              alpha_2=options.alpha_2,
                              lambda_1=options.lambda_1,
                              lambda_2=options.lambda_2,
                              threshold_lambda=options.threshold_lambda,
                              fit_intercept=options.fit_intercept,
                              copy_X=options.copy_X)
    elif options.model == 'gp':
        k_long_term = 66.0**2 * RBF(length_scale=67.0)
        k_seasonal = 2.4**2 * RBF(length_scale=90.0) * ExpSineSquared(
            length_scale=150, periodicity=1.0, periodicity_bounds=(0, 10000))
        k_medium_term = 0.66**2 * RationalQuadratic(length_scale=1.2,
                                                    alpha=0.78)
        k_noise = 0.18**2 * RBF(length_scale=0.134) + WhiteKernel(
            noise_level=0.19**2)
        #kernel_gpml = k_long_term + k_seasonal + k_medium_term + k_noise
        kernel_gpml = k_long_term + k_seasonal + k_medium_term + k_noise

        model = GaussianProcessRegressor(
            kernel=kernel_gpml,  #alpha=0,
            optimizer=None,
            normalize_y=True)
    elif options.model == 'llasso':
        model = LocalizedLasso(num_iter=options.n_loops,
                               batch_size=options.batch_size)
    elif options.model == 'nlasso':
        model = NetworkLasso(num_iter=options.n_loops,
                             batch_size=options.batch_size)

        graph_data = pd.read_csv(options.graph_data,
                                 names=[
                                     'date', 'start_hour', 'src', 'dst',
                                     'type', 'sum_delay', 'sum_ahead',
                                     'add_delay', 'add_ahead', 'train_count'
                                 ])

        #stations_to_pick = options.stations_to_pick.split(',')
        #graph = model.fetch_connections(graph_data, stations_to_pick)
        model.fetch_connections(graph_data)

    if options.pca:
        ipca = IncrementalPCA(n_components=options.pca_components,
                              whiten=options.whiten,
                              copy=False)

    rmses, maes, r2s, skills, start_times, end_times, end_times_obj = [], [], [], [], [], [], []
    X_complete = []  # Used for feature selection

    start = starttime
    end = start + timedelta(days=int(options.day_step),
                            hours=int(options.hour_step))
    if end > endtime: end = endtime

    while end <= endtime and start < end:
        logging.info('Processing time range {} - {}'.format(
            start.strftime('%Y-%m-%d %H:%M'), end.strftime('%Y-%m-%d %H:%M')))

        # Load data ############################################################
        try:
            logging.info('Reading data...')
            data = bq.get_rows(start,
                               end,
                               loc_col='trainstation',
                               project=options.project,
                               dataset=options.feature_dataset,
                               table=options.feature_table,
                               parameters=all_param_names,
                               only_winters=options.only_winters)
            data = io.filter_train_type(labels_df=data,
                                        train_types=options.train_types,
                                        sum_types=True,
                                        train_type_column='train_type',
                                        location_column='trainstation',
                                        time_column='time',
                                        sum_columns=['train_count', 'delay'],
                                        aggs=aggs)

            # Filter only timesteps with large distribution in the whole network
            if options.filter_delay_limit is not None:
                data = io.filter_delay_with_limit(data,
                                                  options.filter_delay_limit)

            if options.y_avg_hours is not None:
                data = io.calc_running_delay_avg(data, options.y_avg_hours)

            if options.y_avg:
                data = io.calc_delay_avg(data)

            data.sort_values(by=['time', 'trainstation'], inplace=True)

            if options.impute:
                logging.info('Imputing missing values...')
                data.drop(columns=['train_type'], inplace=True)
                data = imputer.fit_transform(data)
                data.loc[:, 'train_type'] = None

            if options.month:
                logging.info('Adding month to the dataset...')
                data['month'] = data['time'].map(lambda x: x.month)
                if 'month' not in options.feature_params:
                    options.feature_params.append('month')

            if options.model == 'ard' and len(data) > options.n_samples:
                logging.info('Sampling {} values from data...'.format(
                    options.n_samples))
                data = data.sample(options.n_samples)

            l_data = data.loc[:, options.label_params]
            f_data = data.loc[:, options.feature_params]

        except ValueError as e:
            f_data, l_data = [], []

        if len(f_data) < 2 or len(l_data) < 2:
            start = end
            end = start + timedelta(days=int(options.day_step),
                                    hours=int(options.hour_step))
            continue

        logging.info('Processing {} rows...'.format(len(f_data)))

        train, test = train_test_split(data, test_size=0.1)
        X_train = train.loc[:,
                            options.feature_params].astype(np.float32).values
        y_train = train.loc[:, options.label_params].astype(
            np.float32).values.ravel()
        X_test = test.loc[:, options.feature_params].astype(np.float32).values
        y_test = test.loc[:, options.label_params].astype(
            np.float32).values.ravel()

        logging.debug('Features shape: {}'.format(X_train.shape))

        if options.normalize:
            logging.info('Normalizing data...')
            xscaler, yscaler = StandardScaler(), StandardScaler()

            X_train = xscaler.fit_transform(X_train)
            X_test = xscaler.transform(X_test)

            if len(options.label_params) == 1:
                y_train = yscaler.fit_transform(y_train.reshape(-1, 1)).ravel()
                #y_test = yscaler.transform(y_test.reshape(-1, 1)).ravel()
            else:
                y_train = yscaler.fit_transform(y_train)
                #y_test = yscaler.transform(y_test)

        if options.pca:
            logging.info('Doing PCA analyzis for the data...')
            X_train = ipca.fit_transform(X_train)
            fname = options.output_path + '/ipca_explained_variance.png'
            viz.explained_variance(ipca, fname)
            #io._upload_to_bucket(filename=fname, ext_filename=fname)
            X_test = ipca.fit_transform(X_test)

        if options.model == 'llasso':
            graph_data = pd.read_csv(options.graph_data,
                                     names=[
                                         'date', 'start_hour', 'src', 'dst',
                                         'type', 'sum_delay', 'sum_ahead',
                                         'add_delay', 'add_ahead',
                                         'train_count'
                                     ])
            graph = model.fetch_connections(graph_data)

        logging.debug('Features shape after pre-processing: {}'.format(
            X_train.shape))

        # FIT ##################################################################

        if options.cv:
            logging.info('Doing random search for hyper parameters...')

            if options.model == 'rf':
                param_grid = {
                    "n_estimators": [10, 100, 200, 800],
                    "max_depth": [3, 20, None],
                    "max_features": ["auto", "sqrt", "log2", None],
                    "min_samples_split": [2, 5, 10],
                    "min_samples_leaf": [1, 2, 4, 10],
                    "bootstrap": [True, False]
                }
            elif options.model == 'lr':
                param_grid = {
                    "penalty": [None, 'l2', 'l1'],
                    "alpha": [0.00001, 0.0001, 0.001, 0.01, 0.1],
                    "l1_ratio": [0.1, 0.15, 0.2, 0.5],
                    "shuffle": [True, False],
                    "learning_rate": ['constant', 'optimal', 'invscaling'],
                    "eta0": [0.001, 0.01, 0.1],
                    "power_t": [0.1, 0.25, 0.5]
                }
            elif options.model == 'svr':
                param_grid = {
                    "C": [0.001, 0.01, 0.1, 1, 10],
                    "epsilon": [0.01, 0.1, 0.5],
                    "kernel":
                    ['rbf', 'linear', 'poly', 'sigmoid', 'precomputed'],
                    "degree": [2, 3, 4],
                    "shrinking": [True, False],
                    "gamma": [0.001, 0.01, 0.1],
                    "coef0": [0, 0.1, 1]
                }
            else:
                raise ("No param_grid set for given model ({})".format(
                    options.model))

            random_search = RandomizedSearchCV(model,
                                               param_distributions=param_grid,
                                               n_iter=int(
                                                   options.n_iter_search),
                                               n_jobs=-1)

            random_search.fit(X_train, y_train)
            logging.info("RandomizedSearchCV done.")
            fname = options.output_path + '/random_search_cv_results.txt'
            io.report_cv_results(random_search.cv_results_, fname)
            #io._upload_to_bucket(filename=fname, ext_filename=fname)
            sys.exit()
        else:
            logging.info('Training...')
            if options.model in ['rf', 'svr', 'ard', 'gp']:
                model.fit(X_train, y_train)
                if options.feature_selection:
                    X_complete = X_train
                    y_complete = y_train
                    meta_complete = data.loc[:, options.meta_params]
            elif options.model in ['llasso']:
                model.fit(X_train,
                          y_train,
                          stations=train.loc[:, 'trainstation'].values)
            elif options.model in ['nlasso']:
                model.partial_fit(X_train,
                                  y_train,
                                  stations=train.loc[:, 'trainstation'].values)
            else:
                model.partial_fit(X_train, y_train)
                if options.feature_selection:
                    try:
                        X_complete = np.append(X_complete, X_train)
                        y_complete = np.append(Y_complete, y_train)
                        meta_complete = meta_complete.append(
                            data.loc[:, options.meta_params])
                    except (ValueError, NameError):
                        X_complete = X_train
                        y_complete = y_train
                        meta_complete = data.loc[:, options.meta_params]

        # EVALUATE #############################################################

        # Check training score to estimate amount of overfitting
        # Here we assume that we have a datetime index (from time columns)
        y_pred_train = model.predict(X_train)
        rmse_train = np.sqrt(mean_squared_error(y_train, y_pred_train))
        mae_train = np.sqrt(mean_squared_error(y_train, y_pred_train))
        logging.info('Training data RMSE: {} and MAE: {}'.format(
            rmse_train, mae_train))

        #try:
        if True:
            print(train)
            #range = ('2013-02-01','2013-02-28')
            range = ('2010-01-01', '2010-01-02')
            X_train_sample = train.loc[range[0]:range[1],
                                       options.feature_params].astype(
                                           np.float32).values

            target = train.loc[range[0]:range[1], options.label_params].astype(
                np.float32).values.ravel()
            y_pred_sample = model.predict(X_train_sample)

            times = train.loc[range[0]:range[1], 'time'].values
            df = pd.DataFrame(times + y_pred_sample)
            print(df)
            sys.exit()

            # Draw visualisation
            fname = '{}/timeseries_training_data.png'.format(
                options.output_path)
            viz.plot_delay(times, target, y_pred,
                           'Delay for station {}'.format(stationName), fname)

            fname = '{}/scatter_all_stations.png'.format(options.vis_path)
            viz.scatter_predictions(times,
                                    target,
                                    y_pred,
                                    savepath=options.vis_path,
                                    filename='scatter_{}'.format(station))
        #except KeyError:
        #    pass

        # Mean delay over the whole dataset (both train and validation),
        # used to calculate Brier Skill
        if options.y_avg:
            mean_delay = 3.375953418071136
        else:
            mean_delay = 6.011229358531166

        if options.model == 'llasso':
            print('X_test shape: {}'.format(X_test.shape))
            y_pred, weights = model.predict(X_test,
                                            test.loc[:, 'trainstation'].values)
        else:
            y_pred = model.predict(X_test)

        if options.normalize:
            y_pred = yscaler.inverse_transform(y_pred)

        rmse = np.sqrt(mean_squared_error(y_test, y_pred))
        mae = mean_absolute_error(y_test, y_pred)
        r2 = r2_score(y_test, y_pred)
        rmse_stat = math.sqrt(
            mean_squared_error(y_test, np.full_like(y_test, mean_delay)))
        skill = 1 - rmse / rmse_stat

        rmses.append(rmse)
        maes.append(mae)
        r2s.append(r2)
        skills.append(skill)
        start_times.append(start.strftime('%Y-%m-%dT%H:%M:%S'))
        end_times.append(end.strftime('%Y-%m-%dT%H:%M:%S'))
        end_times_obj.append(end)

        if options.model in ['rf', 'lr', 'ard', 'gp']:
            logging.info('R2 score for training: {}'.format(
                model.score(X_train, y_train)))

        logging.info('RMSE: {}'.format(rmse))
        logging.info('MAE: {}'.format(mae))
        logging.info('R2 score: {}'.format(r2))
        logging.info('Brier Skill Score score: {}'.format(skill))

        start = end
        end = start + timedelta(days=int(options.day_step),
                                hours=int(options.hour_step))
        if end > endtime: end = endtime

    # SAVE #####################################################################
    io.save_scikit_model(model,
                         filename=options.save_file,
                         ext_filename=options.save_file)
    if options.normalize:
        fname = options.save_path + '/xscaler.pkl'
        io.save_scikit_model(xscaler, filename=fname, ext_filename=fname)
        fname = options.save_path + '/yscaler.pkl'
        io.save_scikit_model(yscaler, filename=fname, ext_filename=fname)

    if options.model == 'rf':
        fname = options.output_path + '/rfc_feature_importance.png'
        viz.rfc_feature_importance(model.feature_importances_,
                                   fname,
                                   feature_names=options.feature_params)
        #io._upload_to_bucket(filename=fname, ext_filename=fname)

    try:
        fname = options.output_path + '/learning_over_time.png'
        viz.plot_learning_over_time(end_times_obj,
                                    rmses,
                                    maes,
                                    r2s,
                                    filename=fname)
        #io._upload_to_bucket(filename=fname, ext_filename=fname)
    except Exception as e:
        logging.error(e)

    error_data = {
        'start_times': start_times,
        'end_times': end_times,
        'rmse': rmses,
        'mae': maes,
        'r2': r2s,
        'skill': skills
    }
    fname = '{}/training_time_validation_errors.csv'.format(
        options.output_path)
    io.write_csv(error_data, filename=fname, ext_filename=fname)

    # FEATURE SELECTION ########################################################
    if options.feature_selection:
        logging.info('Doing feature selection...')
        selector = SelectFromModel(model, prefit=True)
        print(pd.DataFrame(data=X_complete))
        X_selected = selector.transform(X_complete)

        selected_columns = f_data.columns.values[selector.get_support()]
        logging.info(
            'Selected following parameters: {}'.format(selected_columns))
        data_sel = meta_complete.join(
            pd.DataFrame(data=y_complete, columns=options.label_params)).join(
                pd.DataFrame(data=X_selected, columns=selected_columns))

        print(pd.DataFrame(data=X_selected, columns=selected_columns))
        print(data_sel)
Ejemplo n.º 2
0
def main():
    """
    Main program
    """

    # Print GPU availability
    local_device_protos = device_lib.list_local_devices()
    logging.info(
        [x.name for x in local_device_protos if x.device_type == 'GPU'])

    bq = BQHandler()
    io = IO(gs_bucket=options.gs_bucket)
    viz = Viz(io)

    starttime, endtime = io.get_dates(options)
    logging.info('Using dataset {}.{} and time range {} - {}'.format(
        options.feature_dataset, options.feature_table,
        starttime.strftime('%Y-%m-%d'), endtime.strftime('%Y-%m-%d')))

    all_param_names = list(
        set(options.label_params + options.feature_params +
            options.meta_params))
    aggs = io.get_aggs_from_param_names(options.feature_params)

    logging.info('Building model...')
    dim = len(options.feature_params)
    if options.month: dim += 1
    model = convlstm.Regression(options, dim).get_model()

    logging.info('Reading data...')
    bq.set_params(batch_size=2500000,
                  loc_col='trainstation',
                  project=options.project,
                  dataset=options.feature_dataset,
                  table=options.feature_table,
                  parameters=all_param_names,
                  locations=options.train_stations,
                  only_winters=options.only_winters,
                  reason_code_table=options.reason_code_table)

    data = bq.get_rows(starttime, endtime)

    data = io.filter_train_type(labels_df=data,
                                train_types=options.train_types,
                                sum_types=True,
                                train_type_column='train_type',
                                location_column='trainstation',
                                time_column='time',
                                sum_columns=['train_count', 'delay'],
                                aggs=aggs)

    if options.y_avg_hours is not None:
        data = io.calc_running_delay_avg(data, options.y_avg_hours)

    if options.y_avg:
        data = io.calc_delay_avg(data)

    data.sort_values(by=['time', 'trainstation'], inplace=True)

    if options.month:
        logging.info('Adding month to the dataset...')
        data['month'] = data['time'].map(lambda x: x.month)
        options.feature_params.append('month')

    if options.normalize:
        logging.info('Normalizing data...')
        xscaler = StandardScaler()
        yscaler = StandardScaler()

        labels = data.loc[:, options.label_params].astype(
            np.float32).values.reshape((-1, 1))
        yscaler.fit(labels)
        scaled_labels = pd.DataFrame(yscaler.transform(labels),
                                     columns=['delay'])

        non_scaled_data = data.loc[:, options.meta_params + ['class']]
        scaled_features = pd.DataFrame(xscaler.fit_transform(
            data.loc[:, options.feature_params]),
                                       columns=options.feature_params)

        data = pd.concat([non_scaled_data, scaled_features, scaled_labels],
                         axis=1)

        fname = options.save_path + '/xscaler.pkl'
        io.save_scikit_model(xscaler, fname, fname)
        fname = options.save_path + '/yscaler.pkl'
        io.save_scikit_model(yscaler, fname, fname)

    if options.pca:
        logging.info('Doing PCA analyzis for the data...')
        ipca = IncrementalPCA(n_components=options.pca_components,
                              whiten=options.whiten,
                              copy=False)

        non_processed_data = data.loc[:, options.meta_params +
                                      options.label_params]
        processed_data = data.loc[:, options.feature_params]
        ipca.fit(processed_data)
        processed_features = pd.DataFrame(ipca.transform(processed_data))

        data = pd.concat([non_processed_data, processed_data], axis=1)

        fname = options.output_path + '/ipca_explained_variance.png'
        viz.explained_variance(ipca, fname)
        io._upload_to_bucket(filename=fname, ext_filename=fname)

    data_train, data_test = train_test_split(data, test_size=0.33)

    # Define model
    batch_size = 512
    logging.info('Batch size: {}'.format(batch_size))

    # Initialization
    losses, val_losses, accs, val_accs, steps = [], [], [], [], []

    boardcb = TensorBoard(log_dir=options.log_dir + '/lstm',
                          histogram_freq=0,
                          write_graph=True,
                          write_images=True)

    logging.info('Data shape: {}'.format(
        data_train.loc[:, options.feature_params].values.shape))

    data_gen = TimeseriesGenerator(
        data_train.loc[:, options.feature_params].values,
        data_train.loc[:, options.label_params].values,
        length=24,
        sampling_rate=1,
        batch_size=batch_size)

    data_test_gen = TimeseriesGenerator(
        data_test.loc[:, options.feature_params].values,
        data_test.loc[:, options.label_params].values,
        length=24,
        sampling_rate=1,
        batch_size=batch_size)

    logging.info('X batch size: {}'.format(data_gen[0][0].shape))
    logging.info('Y batch size: {}'.format(data_gen[1][0].shape))

    history = model.fit_generator(data_gen,
                                  validation_data=data_test_gen,
                                  epochs=options.epochs,
                                  callbacks=[boardcb])  #, batch_size=64)

    history_fname = options.save_path + '/history.pkl'
    io.save_keras_model(options.save_file, history_fname, model,
                        history.history)

    scores = model.evaluate_generator(data_test_gen)
    i = 0
    error_data = {}
    for name in model.metrics_names:
        logging.info('{}: {:.4f}'.format(name, scores[i]))
        error_data[name] = [scores[i]]
        i += 1

    fname = '{}/training_time_validation_errors.csv'.format(
        options.output_path)
    io.write_csv(error_data, filename=fname, ext_filename=fname)

    pred = model.predict_generator(data_test_gen)

    #io.log_class_dist(pred, 4)
    #print(history.history)
    fname = options.output_path + '/learning_over_time.png'
    viz.plot_nn_perf(history.history,
                     metrics={
                         'Error': {
                             'mean_squared_error': 'MSE',
                             'mean_absolute_error': 'MAE'
                         }
                     },
                     filename=fname)
Ejemplo n.º 3
0
def main():
    """
    Get data from db and save it as csv
    """

    bq = BQHandler()
    io = IO(gs_bucket=options.gs_bucket)
    viz = Viz(io=io)
    predictor = Predictor(io, ModelLoader(io), options,
                          options.station_specific_classifier,
                          options.station_specific_regressor)
    predictor.regressor_save_file = options.save_path + '/classifier.pkl'
    predictor.classifier_save_file = options.save_path + '/regressor.pkl'

    # Mean delay over the whole dataset (both train and validation),
    # used to calculate Brier Skill
    mean_delay = options.mean_delay

    starttime, endtime = io.get_dates(options)
    logging.info('Using dataset {} and time range {} - {}'.format(
        options.feature_dataset, starttime.strftime('%Y-%m-%d'),
        endtime.strftime('%Y-%m-%d')))

    # Get params
    all_param_names = list(
        set(options.label_params + options.feature_params +
            options.meta_params + options.classifier_feature_params +
            options.regressor_feature_params))

    # Param list is modified after retrieving data
    classifier_feature_params = copy.deepcopy(
        options.classifier_feature_params)
    regressor_feature_params = copy.deepcopy(options.regressor_feature_params)

    all_feature_params = list(
        set(options.feature_params + options.meta_params +
            options.classifier_feature_params +
            options.regressor_feature_params))
    aggs = io.get_aggs_from_param_names(all_feature_params)

    # Init error dicts
    avg_delay = {}
    avg_pred_delay = {}
    avg_proba = {}
    station_count = 0
    all_times = set()

    station_rmse = {}
    station_mae = {}
    station_r2 = {}
    station_skill = {}

    # For aggregated binary classification metrics
    time_list, target_list, y_pred_bin_list, y_pred_bin_proba_list = [], [], [], []

    # If stations are given as argument use them, else use all stations
    stationList = io.get_train_stations(options.stations_file)
    all_data = None

    if options.locations is not None:
        stations = options.locations
    else:
        stations = stationList.keys()

    # Go through stations
    for station in stations:
        stationName = '{} ({})'.format(stationList[station]['name'], station)
        logging.info('Processing station {}'.format(stationName))

        if hasattr(options, 'classifier_model_file'):
            predictor.classifier_save_file = options.classifier_model_file.replace(
                '{location}', station)
        elif options.station_specific_classifier:
            predictor.classifier_save_file = options.save_path + '/{}'.format(
                station) + '/classifier.pkl'

        if hasattr(options, 'regressor_model_file'):
            predictor.regressor_save_file = options.regressor_model_file.replace(
                '{location}', station)
        elif options.station_specific_regressor:
            predictor.regressor_save_file = options.save_path + '/{}'.format(
                station) + '/regressor.pkl'

        station_rmse[station] = {}
        station_mae[station] = {}
        station_r2[station] = {}
        station_skill[station] = {}

        # Read data and filter desired train types (ic and commuter)
        table = 'features_testset'
        if hasattr(options, 'test_table'):
            table = options.test_table
        data = bq.get_rows(starttime,
                           endtime,
                           loc_col='trainstation',
                           project=options.project,
                           dataset='trains_data',
                           table=table,
                           parameters=all_param_names,
                           only_winters=options.only_winters,
                           reason_code_table=options.reason_code_table,
                           reason_codes_exclude=options.reason_codes_exclude,
                           reason_codes_include=options.reason_codes_include,
                           locations=[station])

        data = io.filter_train_type(labels_df=data,
                                    train_types=['K', 'L'],
                                    sum_types=True,
                                    train_type_column='train_type',
                                    location_column='trainstation',
                                    time_column='time',
                                    sum_columns=['train_count', 'delay'],
                                    aggs=aggs)

        if len(data) == 0:
            continue

        if options.y_avg_hours is not None:
            data = io.calc_running_delay_avg(data, options.y_avg_hours)

        if options.y_avg:
            data = io.calc_delay_avg(data)

        if options.month:
            logging.info('Adding month to the dataset...')
            data = data.assign(
                month=lambda df: df.loc[:, 'time'].map(lambda x: x.month))
            if 'month' not in options.feature_params:
                options.feature_params.append('month')
            if 'month' not in options.regressor_feature_params:
                options.regressor_feature_params.append('month')
            if 'month' not in options.classifier_feature_params:
                options.classifier_feature_params.append('month')

        data.sort_values(by=['time'], inplace=True)
        logging.info('Processing {} rows...'.format(len(data)))

        if all_data is None:
            all_data = data
        else:
            all_data.append(data, ignore_index=True)

        # Pick times for creating error time series
        times = data.loc[:, 'time']
        station_count += 1

        # Run prediction
        try:
            #target, y_pred = predictor.pred(times, data)
            y_pred, y_pred_bin, y_pred_bin_proba = predictor.pred(times, data)
            # Drop first times which LSTM are not able to predict
            #times = times[(len(data)-len(y_pred)):]
        except (PredictionError, ModelError) as e:
            logging.error(e)
            continue

        target = data.loc[:, options.label_params].reset_index(
            drop=True).values.ravel()

        if len(y_pred) < 1 or len(target) < 1:
            continue

        # Create timeseries of predicted and happended delay
        i = 0
        for t in times:
            try:
                if t not in avg_delay.keys():
                    avg_delay[t] = [target[i]]
                    avg_pred_delay[t] = [y_pred[i]]
                    if predictor.y_pred_bin_proba is not None:
                        avg_proba[t] = [predictor.y_pred_bin_proba[i, 1]]
                else:
                    avg_delay[t].append(target[i])
                    avg_pred_delay[t].append(y_pred[i])
                    if predictor.y_pred_bin_proba is not None:
                        avg_proba[t].append(predictor.y_pred_bin_proba[i, 1])
            except IndexError as e:
                # LSTM don't have first time steps because it don't
                # have necessary history
                pass
            i += 1

        # For creating visualisation
        all_times = all_times.union(set(times))

        # If only average plots are asked, continue to next station
        if options.only_avg == 1:
            continue

        # Calculate errors for given station, first for all periods and then for whole time range
        if predictor.y_pred_bin is not None:
            time_list += list(times)

            #feature_list += list()
            target_list += list(target)
            y_pred_bin_list += list(predictor.y_pred_bin)
            y_pred_bin_proba_list += list(predictor.y_pred_bin_proba)

            splits = viz._split_to_parts(list(times), [
                target, y_pred, predictor.y_pred_bin,
                predictor.y_pred_bin_proba
            ], 2592000)
        else:
            splits = viz._split_to_parts(list(times), [target, y_pred],
                                         2592000)

        for i in range(0, len(splits)):

            logging.info('Month {}:'.format(i + 1))

            if predictor.y_pred_bin is not None:
                times_, target_, y_pred_, y_pred_bin_, y_pred_bin_proba_ = splits[
                    i]
                viz.classification_perf_metrics(y_pred_bin_proba_, y_pred_bin_,
                                                target_, options, times_,
                                                station)
            else:
                times_, target_, y_pred_ = splits[i]

            rmse = math.sqrt(metrics.mean_squared_error(target_, y_pred_))
            mae = metrics.mean_absolute_error(target_, y_pred_)
            r2 = metrics.r2_score(target_, y_pred_)
            rmse_stat = math.sqrt(
                metrics.mean_squared_error(target_,
                                           np.full_like(target_, mean_delay)))
            skill = 1 - rmse / rmse_stat

            # Put errors to timeseries
            station_rmse[station][i] = rmse
            station_mae[station][i] = mae
            station_r2[station][i] = r2
            station_skill[station][i] = skill

            logging.info('RMSE of station {} month {}: {:.4f}'.format(
                stationName, i + 1, rmse))
            logging.info('MAE of station {} month {}: {:.4f}'.format(
                stationName, i + 1, mae))
            logging.info('R2 score of station {} month {}: {:.4f}'.format(
                stationName, i + 1, r2))
            logging.info('Skill (RMSE) of station {} month {}: {:.4f}'.format(
                stationName, i + 1, skill))

        mse = math.sqrt(metrics.mean_squared_error(target, y_pred))
        mae = metrics.mean_absolute_error(target, y_pred)
        r2 = metrics.r2_score(target, y_pred)
        rmse_stat = math.sqrt(
            metrics.mean_squared_error(target,
                                       np.full_like(target, mean_delay)))
        skill = 1 - rmse / rmse_stat

        station_rmse[station]['all'] = rmse
        station_mae[station]['all'] = mae
        station_r2[station]['all'] = r2
        station_skill[station]['all'] = skill

        logging.info('All periods:')
        logging.info('RMSE of station {} month {}: {:.4f}'.format(
            stationName, i + 1, rmse))
        logging.info('MAE of station {} month {}: {:.4f}'.format(
            stationName, i + 1, mae))
        logging.info('R2 score of station {} month {}: {:.4f}'.format(
            stationName, i + 1, r2))
        logging.info('Skill (RMSE) of station {} month {}: {:.4f}'.format(
            stationName, i + 1, skill))

        # Create csv and upload it to pucket
        times_formatted = [t.strftime('%Y-%m-%dT%H:%M:%S') for t in times]
        delay_data = {
            'times': times_formatted,
            'delay': target,
            'predicted delay': y_pred
        }
        fname = '{}/delays_{}.csv'.format(options.vis_path, station)
        io.write_csv(delay_data, fname, fname)

        # Draw visualisation
        if predictor.y_pred_bin_proba is not None:
            fname = '{}/timeseries_proba_{}'.format(options.vis_path, station)
            proba = predictor.y_pred_bin_proba[:, 1]
            viz.plot_delay(times,
                           target,
                           None,
                           'Delay for station {}'.format(stationName),
                           fname,
                           all_proba=proba,
                           proba_mode='same',
                           color_threshold=options.class_limit)
        #else:
        fname = '{}/timeseries_regression_{}'.format(options.vis_path, station)
        viz.plot_delay(times,
                       target,
                       y_pred,
                       'Delay for station {}'.format(stationName),
                       fname,
                       all_proba=None)

        fname = '{}/scatter_all_stations.png'.format(options.vis_path)
        viz.scatter_predictions(times,
                                target,
                                y_pred,
                                savepath=options.vis_path,
                                filename='scatter_{}'.format(station))

    # Save all station related results to csv and upload them to bucket
    fname = '{}/station_rmse.csv'.format(options.vis_path)
    io.dict_to_csv(station_rmse, fname, fname)
    fname = '{}/station_mae.csv'.format(options.vis_path)
    io.dict_to_csv(station_mae, fname, fname)
    fname = '{}/station_r2.csv'.format(options.vis_path)
    io.dict_to_csv(station_r2, fname, fname)
    fname = '{}/station_skill_rmse.csv'.format(options.vis_path)
    io.dict_to_csv(station_skill, fname, fname)

    # Create timeseries of avg actual delay and predicted delay
    all_times = sorted(list(all_times))
    for t, l in avg_delay.items():
        avg_delay[t] = sum(l) / len(l)
    for t, l in avg_pred_delay.items():
        avg_pred_delay[t] = sum(l) / len(l)
    for t, l in avg_proba.items():
        avg_proba[t] = sum(l) / len(l)

    avg_delay = list(
        OrderedDict(sorted(avg_delay.items(), key=lambda t: t[0])).values())
    avg_pred_delay = list(
        OrderedDict(sorted(avg_pred_delay.items(),
                           key=lambda t: t[0])).values())
    avg_proba = list(
        OrderedDict(sorted(avg_proba.items(), key=lambda t: t[0])).values())

    # Calculate average over all times and stations, first for all months separately, then for whole time range
    splits = viz._split_to_parts(list(times), [avg_delay, avg_pred_delay],
                                 2592000)

    for i in range(0, len(splits)):
        times_, avg_delay_, avg_pred_delay_ = splits[i]

        try:
            rmse = math.sqrt(
                metrics.mean_squared_error(avg_delay_, avg_pred_delay_))
            mae = metrics.mean_absolute_error(avg_delay_, avg_pred_delay_)
            r2 = metrics.r2_score(avg_delay_, avg_pred_delay_)
            rmse_stat = math.sqrt(
                metrics.mean_squared_error(
                    avg_delay_, np.full_like(avg_delay_, mean_delay)))
            skill = 1 - rmse / rmse_stat
        except ValueError:
            logging.warning('Zero samples in some class')
            continue

        logging.info('Month: {}'.format(i + 1))
        logging.info(
            'RMSE of average delay over all stations: {:.4f}'.format(rmse))
        logging.info(
            'MAE of average delay over all stations: {:.4f}'.format(mae))
        logging.info(
            'R2 score of average delay over all stations: {:.4f}'.format(r2))
        logging.info(
            'Skill score (RMSE) of average delay over all stations: {:.4f}'.
            format(skill))

        # Write average data into file
        avg_errors = {
            'rmse': rmse,
            'mae': mae,
            'r2': r2,
            'skill': skill,
            'nro_of_samples': len(avg_delay)
        }
        fname = '{}/avg_erros_{}.csv'.format(options.vis_path, i)
        io.dict_to_csv(avg_errors, fname, fname)

    rmse = math.sqrt(metrics.mean_squared_error(avg_delay, avg_pred_delay))
    #rmse_mean = np.mean(list(station_rmse.values()))
    mae = metrics.mean_absolute_error(avg_delay, avg_pred_delay)
    #mae_mean = np.mean(list(station_mae.values()))
    r2 = metrics.r2_score(avg_delay, avg_pred_delay)
    rmse_stat = math.sqrt(
        metrics.mean_squared_error(avg_delay,
                                   np.full_like(avg_delay, mean_delay)))
    skill = 1 - rmse / rmse_stat
    #skill_mean = 1 - rmse_mean/rmse_stat

    logging.info('All periods:')
    logging.info(
        'RMSE of average delay over all stations: {:.4f}'.format(rmse))
    #logging.info('Average RMSE of all station RMSEs: {:.4f}'.format(rmse_mean))
    logging.info('MAE of average delay over all stations: {:.4f}'.format(mae))
    #logging.info('Average MAE of all station MAEs: {:.4f}'.format(mae_mean))
    logging.info(
        'R2 score of average delay over all stations: {:.4f}'.format(r2))
    logging.info(
        'Skill score (RMSE) of average delay over all stations: {:.4f}'.format(
            skill))
    #logging.info('Skill score (avg RMSE) of all stations: {:.4f}'.format(skill_mean))

    # Write average data into file
    avg_errors = {
        'rmse': rmse,
        'mae': mae,
        'r2': r2,
        #'rmse_mean': rmse_mean,
        #'mae_mean': mae_mean,
        'skill': skill,
        #'skill_mean': skill_mean,
        'nro_of_samples': len(avg_delay)
    }
    fname = '{}/avg_erros.csv'.format(options.vis_path)
    io.dict_to_csv(avg_errors, fname, fname)

    # Create timeseries of average delay and predicted delays over all stations
    all_times_formatted = [t.strftime('%Y-%m-%dT%H:%M:%S') for t in all_times]
    delay_data = {
        'times': all_times_formatted,
        'delay': avg_delay,
        'predicted delay': avg_pred_delay
    }

    # write csv
    fname = '{}/avg_delays_all_stations.csv'.format(options.vis_path)
    io.write_csv(delay_data, fname, fname)

    # visualise
    if not avg_proba:
        proba = None
    else:
        proba = avg_proba
    fname = '{}/timeseries_avg_all_stations.png'.format(options.vis_path)

    if predictor.y_pred_bin is not None:
        viz.plot_delay(all_times,
                       avg_delay,
                       None,
                       'Average delay for all station',
                       fname,
                       all_proba=proba,
                       proba_mode='same',
                       color_threshold=options.class_limit)
    else:
        viz.plot_delay(all_times, avg_delay, avg_pred_delay,
                       'Average delay for all station', fname)

    fname = '{}/scatter_all_stations.png'.format(options.vis_path)
    viz.scatter_predictions(all_times,
                            avg_delay,
                            avg_pred_delay,
                            savepath=options.vis_path,
                            filename='scatter_all_stations')

    # Binary classification metrics
    if predictor.y_pred_bin is not None:
        all_data.sort_values(by=['time'], inplace=True)
        times = all_data.loc[:, 'time'].values
        try:
            target = all_data.loc[:, options.label_params].reset_index(
                drop=True).values.ravel()
            y_pred, y_pred_bin, y_pred_bin_proba = predictor.pred(
                times, all_data)
            # Drop first times which LSTM are not able to predict
            times = times[(len(all_data) - len(y_pred)):]
            splits = viz._split_to_parts(list(times), [
                target, y_pred, predictor.y_pred_bin,
                predictor.y_pred_bin_proba
            ], 2592000)

            for i in range(0, len(splits)):
                #times_, target_, y_pred_bin_, y_pred_bin_proba_ = splits[i]
                times_, target_, y_pred_, y_pred_bin_, y_pred_bin_proba_ = splits[
                    i]
                viz.classification_perf_metrics(y_pred_bin_proba_, y_pred_bin_,
                                                target_, options, times_,
                                                'all')
        except (PredictionError, ModelError) as e:
            logging.error(e)
            pass