def main(): """ Get data from db and save it as csv """ bq = BQHandler() io = IO(gs_bucket=options.gs_bucket) viz = Viz(io=io) starttime, endtime = io.get_dates(options) logging.info('Using dataset {} and time range {} - {}'.format( options.feature_dataset, starttime.strftime('%Y-%m-%d'), endtime.strftime('%Y-%m-%d'))) all_param_names = options.label_params + options.feature_params + options.meta_params aggs = io.get_aggs_from_param_names(options.feature_params) if options.model == 'rf': model = RandomForestRegressor( n_estimators=options.n_estimators, n_jobs=-1, min_samples_leaf=options.min_samples_leaf, min_samples_split=options.min_samples_split, max_features=options.max_features, max_depth=options.max_depth, bootstrap=options.bootstrap) elif options.model == 'lr': model = SGDRegressor(warm_start=True, max_iter=options.n_loops, shuffle=options.shuffle, power_t=options.power_t, penalty=options.regularizer, learning_rate=options.learning_rate, eta0=options.eta0, alpha=options.alpha, tol=0.0001) elif options.model == 'svr': model = SVR() elif options.model == 'ard': model = ARDRegression(n_iter=options.n_loops, alpha_1=options.alpha_1, alpha_2=options.alpha_2, lambda_1=options.lambda_1, lambda_2=options.lambda_2, threshold_lambda=options.threshold_lambda, fit_intercept=options.fit_intercept, copy_X=options.copy_X) elif options.model == 'gp': k_long_term = 66.0**2 * RBF(length_scale=67.0) k_seasonal = 2.4**2 * RBF(length_scale=90.0) * ExpSineSquared( length_scale=150, periodicity=1.0, periodicity_bounds=(0, 10000)) k_medium_term = 0.66**2 * RationalQuadratic(length_scale=1.2, alpha=0.78) k_noise = 0.18**2 * RBF(length_scale=0.134) + WhiteKernel( noise_level=0.19**2) #kernel_gpml = k_long_term + k_seasonal + k_medium_term + k_noise kernel_gpml = k_long_term + k_seasonal + k_medium_term + k_noise model = GaussianProcessRegressor( kernel=kernel_gpml, #alpha=0, optimizer=None, normalize_y=True) elif options.model == 'llasso': model = LocalizedLasso(num_iter=options.n_loops, batch_size=options.batch_size) elif options.model == 'nlasso': model = NetworkLasso(num_iter=options.n_loops, batch_size=options.batch_size) graph_data = pd.read_csv(options.graph_data, names=[ 'date', 'start_hour', 'src', 'dst', 'type', 'sum_delay', 'sum_ahead', 'add_delay', 'add_ahead', 'train_count' ]) #stations_to_pick = options.stations_to_pick.split(',') #graph = model.fetch_connections(graph_data, stations_to_pick) model.fetch_connections(graph_data) if options.pca: ipca = IncrementalPCA(n_components=options.pca_components, whiten=options.whiten, copy=False) rmses, maes, r2s, skills, start_times, end_times, end_times_obj = [], [], [], [], [], [], [] X_complete = [] # Used for feature selection start = starttime end = start + timedelta(days=int(options.day_step), hours=int(options.hour_step)) if end > endtime: end = endtime while end <= endtime and start < end: logging.info('Processing time range {} - {}'.format( start.strftime('%Y-%m-%d %H:%M'), end.strftime('%Y-%m-%d %H:%M'))) # Load data ############################################################ try: logging.info('Reading data...') data = bq.get_rows(start, end, loc_col='trainstation', project=options.project, dataset=options.feature_dataset, table=options.feature_table, parameters=all_param_names, only_winters=options.only_winters) data = io.filter_train_type(labels_df=data, train_types=options.train_types, sum_types=True, train_type_column='train_type', location_column='trainstation', time_column='time', sum_columns=['train_count', 'delay'], aggs=aggs) # Filter only timesteps with large distribution in the whole network if options.filter_delay_limit is not None: data = io.filter_delay_with_limit(data, options.filter_delay_limit) if options.y_avg_hours is not None: data = io.calc_running_delay_avg(data, options.y_avg_hours) if options.y_avg: data = io.calc_delay_avg(data) data.sort_values(by=['time', 'trainstation'], inplace=True) if options.impute: logging.info('Imputing missing values...') data.drop(columns=['train_type'], inplace=True) data = imputer.fit_transform(data) data.loc[:, 'train_type'] = None if options.month: logging.info('Adding month to the dataset...') data['month'] = data['time'].map(lambda x: x.month) if 'month' not in options.feature_params: options.feature_params.append('month') if options.model == 'ard' and len(data) > options.n_samples: logging.info('Sampling {} values from data...'.format( options.n_samples)) data = data.sample(options.n_samples) l_data = data.loc[:, options.label_params] f_data = data.loc[:, options.feature_params] except ValueError as e: f_data, l_data = [], [] if len(f_data) < 2 or len(l_data) < 2: start = end end = start + timedelta(days=int(options.day_step), hours=int(options.hour_step)) continue logging.info('Processing {} rows...'.format(len(f_data))) train, test = train_test_split(data, test_size=0.1) X_train = train.loc[:, options.feature_params].astype(np.float32).values y_train = train.loc[:, options.label_params].astype( np.float32).values.ravel() X_test = test.loc[:, options.feature_params].astype(np.float32).values y_test = test.loc[:, options.label_params].astype( np.float32).values.ravel() logging.debug('Features shape: {}'.format(X_train.shape)) if options.normalize: logging.info('Normalizing data...') xscaler, yscaler = StandardScaler(), StandardScaler() X_train = xscaler.fit_transform(X_train) X_test = xscaler.transform(X_test) if len(options.label_params) == 1: y_train = yscaler.fit_transform(y_train.reshape(-1, 1)).ravel() #y_test = yscaler.transform(y_test.reshape(-1, 1)).ravel() else: y_train = yscaler.fit_transform(y_train) #y_test = yscaler.transform(y_test) if options.pca: logging.info('Doing PCA analyzis for the data...') X_train = ipca.fit_transform(X_train) fname = options.output_path + '/ipca_explained_variance.png' viz.explained_variance(ipca, fname) #io._upload_to_bucket(filename=fname, ext_filename=fname) X_test = ipca.fit_transform(X_test) if options.model == 'llasso': graph_data = pd.read_csv(options.graph_data, names=[ 'date', 'start_hour', 'src', 'dst', 'type', 'sum_delay', 'sum_ahead', 'add_delay', 'add_ahead', 'train_count' ]) graph = model.fetch_connections(graph_data) logging.debug('Features shape after pre-processing: {}'.format( X_train.shape)) # FIT ################################################################## if options.cv: logging.info('Doing random search for hyper parameters...') if options.model == 'rf': param_grid = { "n_estimators": [10, 100, 200, 800], "max_depth": [3, 20, None], "max_features": ["auto", "sqrt", "log2", None], "min_samples_split": [2, 5, 10], "min_samples_leaf": [1, 2, 4, 10], "bootstrap": [True, False] } elif options.model == 'lr': param_grid = { "penalty": [None, 'l2', 'l1'], "alpha": [0.00001, 0.0001, 0.001, 0.01, 0.1], "l1_ratio": [0.1, 0.15, 0.2, 0.5], "shuffle": [True, False], "learning_rate": ['constant', 'optimal', 'invscaling'], "eta0": [0.001, 0.01, 0.1], "power_t": [0.1, 0.25, 0.5] } elif options.model == 'svr': param_grid = { "C": [0.001, 0.01, 0.1, 1, 10], "epsilon": [0.01, 0.1, 0.5], "kernel": ['rbf', 'linear', 'poly', 'sigmoid', 'precomputed'], "degree": [2, 3, 4], "shrinking": [True, False], "gamma": [0.001, 0.01, 0.1], "coef0": [0, 0.1, 1] } else: raise ("No param_grid set for given model ({})".format( options.model)) random_search = RandomizedSearchCV(model, param_distributions=param_grid, n_iter=int( options.n_iter_search), n_jobs=-1) random_search.fit(X_train, y_train) logging.info("RandomizedSearchCV done.") fname = options.output_path + '/random_search_cv_results.txt' io.report_cv_results(random_search.cv_results_, fname) #io._upload_to_bucket(filename=fname, ext_filename=fname) sys.exit() else: logging.info('Training...') if options.model in ['rf', 'svr', 'ard', 'gp']: model.fit(X_train, y_train) if options.feature_selection: X_complete = X_train y_complete = y_train meta_complete = data.loc[:, options.meta_params] elif options.model in ['llasso']: model.fit(X_train, y_train, stations=train.loc[:, 'trainstation'].values) elif options.model in ['nlasso']: model.partial_fit(X_train, y_train, stations=train.loc[:, 'trainstation'].values) else: model.partial_fit(X_train, y_train) if options.feature_selection: try: X_complete = np.append(X_complete, X_train) y_complete = np.append(Y_complete, y_train) meta_complete = meta_complete.append( data.loc[:, options.meta_params]) except (ValueError, NameError): X_complete = X_train y_complete = y_train meta_complete = data.loc[:, options.meta_params] # EVALUATE ############################################################# # Check training score to estimate amount of overfitting # Here we assume that we have a datetime index (from time columns) y_pred_train = model.predict(X_train) rmse_train = np.sqrt(mean_squared_error(y_train, y_pred_train)) mae_train = np.sqrt(mean_squared_error(y_train, y_pred_train)) logging.info('Training data RMSE: {} and MAE: {}'.format( rmse_train, mae_train)) #try: if True: print(train) #range = ('2013-02-01','2013-02-28') range = ('2010-01-01', '2010-01-02') X_train_sample = train.loc[range[0]:range[1], options.feature_params].astype( np.float32).values target = train.loc[range[0]:range[1], options.label_params].astype( np.float32).values.ravel() y_pred_sample = model.predict(X_train_sample) times = train.loc[range[0]:range[1], 'time'].values df = pd.DataFrame(times + y_pred_sample) print(df) sys.exit() # Draw visualisation fname = '{}/timeseries_training_data.png'.format( options.output_path) viz.plot_delay(times, target, y_pred, 'Delay for station {}'.format(stationName), fname) fname = '{}/scatter_all_stations.png'.format(options.vis_path) viz.scatter_predictions(times, target, y_pred, savepath=options.vis_path, filename='scatter_{}'.format(station)) #except KeyError: # pass # Mean delay over the whole dataset (both train and validation), # used to calculate Brier Skill if options.y_avg: mean_delay = 3.375953418071136 else: mean_delay = 6.011229358531166 if options.model == 'llasso': print('X_test shape: {}'.format(X_test.shape)) y_pred, weights = model.predict(X_test, test.loc[:, 'trainstation'].values) else: y_pred = model.predict(X_test) if options.normalize: y_pred = yscaler.inverse_transform(y_pred) rmse = np.sqrt(mean_squared_error(y_test, y_pred)) mae = mean_absolute_error(y_test, y_pred) r2 = r2_score(y_test, y_pred) rmse_stat = math.sqrt( mean_squared_error(y_test, np.full_like(y_test, mean_delay))) skill = 1 - rmse / rmse_stat rmses.append(rmse) maes.append(mae) r2s.append(r2) skills.append(skill) start_times.append(start.strftime('%Y-%m-%dT%H:%M:%S')) end_times.append(end.strftime('%Y-%m-%dT%H:%M:%S')) end_times_obj.append(end) if options.model in ['rf', 'lr', 'ard', 'gp']: logging.info('R2 score for training: {}'.format( model.score(X_train, y_train))) logging.info('RMSE: {}'.format(rmse)) logging.info('MAE: {}'.format(mae)) logging.info('R2 score: {}'.format(r2)) logging.info('Brier Skill Score score: {}'.format(skill)) start = end end = start + timedelta(days=int(options.day_step), hours=int(options.hour_step)) if end > endtime: end = endtime # SAVE ##################################################################### io.save_scikit_model(model, filename=options.save_file, ext_filename=options.save_file) if options.normalize: fname = options.save_path + '/xscaler.pkl' io.save_scikit_model(xscaler, filename=fname, ext_filename=fname) fname = options.save_path + '/yscaler.pkl' io.save_scikit_model(yscaler, filename=fname, ext_filename=fname) if options.model == 'rf': fname = options.output_path + '/rfc_feature_importance.png' viz.rfc_feature_importance(model.feature_importances_, fname, feature_names=options.feature_params) #io._upload_to_bucket(filename=fname, ext_filename=fname) try: fname = options.output_path + '/learning_over_time.png' viz.plot_learning_over_time(end_times_obj, rmses, maes, r2s, filename=fname) #io._upload_to_bucket(filename=fname, ext_filename=fname) except Exception as e: logging.error(e) error_data = { 'start_times': start_times, 'end_times': end_times, 'rmse': rmses, 'mae': maes, 'r2': r2s, 'skill': skills } fname = '{}/training_time_validation_errors.csv'.format( options.output_path) io.write_csv(error_data, filename=fname, ext_filename=fname) # FEATURE SELECTION ######################################################## if options.feature_selection: logging.info('Doing feature selection...') selector = SelectFromModel(model, prefit=True) print(pd.DataFrame(data=X_complete)) X_selected = selector.transform(X_complete) selected_columns = f_data.columns.values[selector.get_support()] logging.info( 'Selected following parameters: {}'.format(selected_columns)) data_sel = meta_complete.join( pd.DataFrame(data=y_complete, columns=options.label_params)).join( pd.DataFrame(data=X_selected, columns=selected_columns)) print(pd.DataFrame(data=X_selected, columns=selected_columns)) print(data_sel)
def main(): """ Main program """ # Print GPU availability local_device_protos = device_lib.list_local_devices() logging.info( [x.name for x in local_device_protos if x.device_type == 'GPU']) bq = BQHandler() io = IO(gs_bucket=options.gs_bucket) viz = Viz(io) starttime, endtime = io.get_dates(options) logging.info('Using dataset {}.{} and time range {} - {}'.format( options.feature_dataset, options.feature_table, starttime.strftime('%Y-%m-%d'), endtime.strftime('%Y-%m-%d'))) all_param_names = list( set(options.label_params + options.feature_params + options.meta_params)) aggs = io.get_aggs_from_param_names(options.feature_params) logging.info('Building model...') dim = len(options.feature_params) if options.month: dim += 1 model = convlstm.Regression(options, dim).get_model() logging.info('Reading data...') bq.set_params(batch_size=2500000, loc_col='trainstation', project=options.project, dataset=options.feature_dataset, table=options.feature_table, parameters=all_param_names, locations=options.train_stations, only_winters=options.only_winters, reason_code_table=options.reason_code_table) data = bq.get_rows(starttime, endtime) data = io.filter_train_type(labels_df=data, train_types=options.train_types, sum_types=True, train_type_column='train_type', location_column='trainstation', time_column='time', sum_columns=['train_count', 'delay'], aggs=aggs) if options.y_avg_hours is not None: data = io.calc_running_delay_avg(data, options.y_avg_hours) if options.y_avg: data = io.calc_delay_avg(data) data.sort_values(by=['time', 'trainstation'], inplace=True) if options.month: logging.info('Adding month to the dataset...') data['month'] = data['time'].map(lambda x: x.month) options.feature_params.append('month') if options.normalize: logging.info('Normalizing data...') xscaler = StandardScaler() yscaler = StandardScaler() labels = data.loc[:, options.label_params].astype( np.float32).values.reshape((-1, 1)) yscaler.fit(labels) scaled_labels = pd.DataFrame(yscaler.transform(labels), columns=['delay']) non_scaled_data = data.loc[:, options.meta_params + ['class']] scaled_features = pd.DataFrame(xscaler.fit_transform( data.loc[:, options.feature_params]), columns=options.feature_params) data = pd.concat([non_scaled_data, scaled_features, scaled_labels], axis=1) fname = options.save_path + '/xscaler.pkl' io.save_scikit_model(xscaler, fname, fname) fname = options.save_path + '/yscaler.pkl' io.save_scikit_model(yscaler, fname, fname) if options.pca: logging.info('Doing PCA analyzis for the data...') ipca = IncrementalPCA(n_components=options.pca_components, whiten=options.whiten, copy=False) non_processed_data = data.loc[:, options.meta_params + options.label_params] processed_data = data.loc[:, options.feature_params] ipca.fit(processed_data) processed_features = pd.DataFrame(ipca.transform(processed_data)) data = pd.concat([non_processed_data, processed_data], axis=1) fname = options.output_path + '/ipca_explained_variance.png' viz.explained_variance(ipca, fname) io._upload_to_bucket(filename=fname, ext_filename=fname) data_train, data_test = train_test_split(data, test_size=0.33) # Define model batch_size = 512 logging.info('Batch size: {}'.format(batch_size)) # Initialization losses, val_losses, accs, val_accs, steps = [], [], [], [], [] boardcb = TensorBoard(log_dir=options.log_dir + '/lstm', histogram_freq=0, write_graph=True, write_images=True) logging.info('Data shape: {}'.format( data_train.loc[:, options.feature_params].values.shape)) data_gen = TimeseriesGenerator( data_train.loc[:, options.feature_params].values, data_train.loc[:, options.label_params].values, length=24, sampling_rate=1, batch_size=batch_size) data_test_gen = TimeseriesGenerator( data_test.loc[:, options.feature_params].values, data_test.loc[:, options.label_params].values, length=24, sampling_rate=1, batch_size=batch_size) logging.info('X batch size: {}'.format(data_gen[0][0].shape)) logging.info('Y batch size: {}'.format(data_gen[1][0].shape)) history = model.fit_generator(data_gen, validation_data=data_test_gen, epochs=options.epochs, callbacks=[boardcb]) #, batch_size=64) history_fname = options.save_path + '/history.pkl' io.save_keras_model(options.save_file, history_fname, model, history.history) scores = model.evaluate_generator(data_test_gen) i = 0 error_data = {} for name in model.metrics_names: logging.info('{}: {:.4f}'.format(name, scores[i])) error_data[name] = [scores[i]] i += 1 fname = '{}/training_time_validation_errors.csv'.format( options.output_path) io.write_csv(error_data, filename=fname, ext_filename=fname) pred = model.predict_generator(data_test_gen) #io.log_class_dist(pred, 4) #print(history.history) fname = options.output_path + '/learning_over_time.png' viz.plot_nn_perf(history.history, metrics={ 'Error': { 'mean_squared_error': 'MSE', 'mean_absolute_error': 'MAE' } }, filename=fname)
def main(): """ Get data from db and save it as csv """ bq = BQHandler() io = IO(gs_bucket=options.gs_bucket) viz = Viz(io=io) predictor = Predictor(io, ModelLoader(io), options, options.station_specific_classifier, options.station_specific_regressor) predictor.regressor_save_file = options.save_path + '/classifier.pkl' predictor.classifier_save_file = options.save_path + '/regressor.pkl' # Mean delay over the whole dataset (both train and validation), # used to calculate Brier Skill mean_delay = options.mean_delay starttime, endtime = io.get_dates(options) logging.info('Using dataset {} and time range {} - {}'.format( options.feature_dataset, starttime.strftime('%Y-%m-%d'), endtime.strftime('%Y-%m-%d'))) # Get params all_param_names = list( set(options.label_params + options.feature_params + options.meta_params + options.classifier_feature_params + options.regressor_feature_params)) # Param list is modified after retrieving data classifier_feature_params = copy.deepcopy( options.classifier_feature_params) regressor_feature_params = copy.deepcopy(options.regressor_feature_params) all_feature_params = list( set(options.feature_params + options.meta_params + options.classifier_feature_params + options.regressor_feature_params)) aggs = io.get_aggs_from_param_names(all_feature_params) # Init error dicts avg_delay = {} avg_pred_delay = {} avg_proba = {} station_count = 0 all_times = set() station_rmse = {} station_mae = {} station_r2 = {} station_skill = {} # For aggregated binary classification metrics time_list, target_list, y_pred_bin_list, y_pred_bin_proba_list = [], [], [], [] # If stations are given as argument use them, else use all stations stationList = io.get_train_stations(options.stations_file) all_data = None if options.locations is not None: stations = options.locations else: stations = stationList.keys() # Go through stations for station in stations: stationName = '{} ({})'.format(stationList[station]['name'], station) logging.info('Processing station {}'.format(stationName)) if hasattr(options, 'classifier_model_file'): predictor.classifier_save_file = options.classifier_model_file.replace( '{location}', station) elif options.station_specific_classifier: predictor.classifier_save_file = options.save_path + '/{}'.format( station) + '/classifier.pkl' if hasattr(options, 'regressor_model_file'): predictor.regressor_save_file = options.regressor_model_file.replace( '{location}', station) elif options.station_specific_regressor: predictor.regressor_save_file = options.save_path + '/{}'.format( station) + '/regressor.pkl' station_rmse[station] = {} station_mae[station] = {} station_r2[station] = {} station_skill[station] = {} # Read data and filter desired train types (ic and commuter) table = 'features_testset' if hasattr(options, 'test_table'): table = options.test_table data = bq.get_rows(starttime, endtime, loc_col='trainstation', project=options.project, dataset='trains_data', table=table, parameters=all_param_names, only_winters=options.only_winters, reason_code_table=options.reason_code_table, reason_codes_exclude=options.reason_codes_exclude, reason_codes_include=options.reason_codes_include, locations=[station]) data = io.filter_train_type(labels_df=data, train_types=['K', 'L'], sum_types=True, train_type_column='train_type', location_column='trainstation', time_column='time', sum_columns=['train_count', 'delay'], aggs=aggs) if len(data) == 0: continue if options.y_avg_hours is not None: data = io.calc_running_delay_avg(data, options.y_avg_hours) if options.y_avg: data = io.calc_delay_avg(data) if options.month: logging.info('Adding month to the dataset...') data = data.assign( month=lambda df: df.loc[:, 'time'].map(lambda x: x.month)) if 'month' not in options.feature_params: options.feature_params.append('month') if 'month' not in options.regressor_feature_params: options.regressor_feature_params.append('month') if 'month' not in options.classifier_feature_params: options.classifier_feature_params.append('month') data.sort_values(by=['time'], inplace=True) logging.info('Processing {} rows...'.format(len(data))) if all_data is None: all_data = data else: all_data.append(data, ignore_index=True) # Pick times for creating error time series times = data.loc[:, 'time'] station_count += 1 # Run prediction try: #target, y_pred = predictor.pred(times, data) y_pred, y_pred_bin, y_pred_bin_proba = predictor.pred(times, data) # Drop first times which LSTM are not able to predict #times = times[(len(data)-len(y_pred)):] except (PredictionError, ModelError) as e: logging.error(e) continue target = data.loc[:, options.label_params].reset_index( drop=True).values.ravel() if len(y_pred) < 1 or len(target) < 1: continue # Create timeseries of predicted and happended delay i = 0 for t in times: try: if t not in avg_delay.keys(): avg_delay[t] = [target[i]] avg_pred_delay[t] = [y_pred[i]] if predictor.y_pred_bin_proba is not None: avg_proba[t] = [predictor.y_pred_bin_proba[i, 1]] else: avg_delay[t].append(target[i]) avg_pred_delay[t].append(y_pred[i]) if predictor.y_pred_bin_proba is not None: avg_proba[t].append(predictor.y_pred_bin_proba[i, 1]) except IndexError as e: # LSTM don't have first time steps because it don't # have necessary history pass i += 1 # For creating visualisation all_times = all_times.union(set(times)) # If only average plots are asked, continue to next station if options.only_avg == 1: continue # Calculate errors for given station, first for all periods and then for whole time range if predictor.y_pred_bin is not None: time_list += list(times) #feature_list += list() target_list += list(target) y_pred_bin_list += list(predictor.y_pred_bin) y_pred_bin_proba_list += list(predictor.y_pred_bin_proba) splits = viz._split_to_parts(list(times), [ target, y_pred, predictor.y_pred_bin, predictor.y_pred_bin_proba ], 2592000) else: splits = viz._split_to_parts(list(times), [target, y_pred], 2592000) for i in range(0, len(splits)): logging.info('Month {}:'.format(i + 1)) if predictor.y_pred_bin is not None: times_, target_, y_pred_, y_pred_bin_, y_pred_bin_proba_ = splits[ i] viz.classification_perf_metrics(y_pred_bin_proba_, y_pred_bin_, target_, options, times_, station) else: times_, target_, y_pred_ = splits[i] rmse = math.sqrt(metrics.mean_squared_error(target_, y_pred_)) mae = metrics.mean_absolute_error(target_, y_pred_) r2 = metrics.r2_score(target_, y_pred_) rmse_stat = math.sqrt( metrics.mean_squared_error(target_, np.full_like(target_, mean_delay))) skill = 1 - rmse / rmse_stat # Put errors to timeseries station_rmse[station][i] = rmse station_mae[station][i] = mae station_r2[station][i] = r2 station_skill[station][i] = skill logging.info('RMSE of station {} month {}: {:.4f}'.format( stationName, i + 1, rmse)) logging.info('MAE of station {} month {}: {:.4f}'.format( stationName, i + 1, mae)) logging.info('R2 score of station {} month {}: {:.4f}'.format( stationName, i + 1, r2)) logging.info('Skill (RMSE) of station {} month {}: {:.4f}'.format( stationName, i + 1, skill)) mse = math.sqrt(metrics.mean_squared_error(target, y_pred)) mae = metrics.mean_absolute_error(target, y_pred) r2 = metrics.r2_score(target, y_pred) rmse_stat = math.sqrt( metrics.mean_squared_error(target, np.full_like(target, mean_delay))) skill = 1 - rmse / rmse_stat station_rmse[station]['all'] = rmse station_mae[station]['all'] = mae station_r2[station]['all'] = r2 station_skill[station]['all'] = skill logging.info('All periods:') logging.info('RMSE of station {} month {}: {:.4f}'.format( stationName, i + 1, rmse)) logging.info('MAE of station {} month {}: {:.4f}'.format( stationName, i + 1, mae)) logging.info('R2 score of station {} month {}: {:.4f}'.format( stationName, i + 1, r2)) logging.info('Skill (RMSE) of station {} month {}: {:.4f}'.format( stationName, i + 1, skill)) # Create csv and upload it to pucket times_formatted = [t.strftime('%Y-%m-%dT%H:%M:%S') for t in times] delay_data = { 'times': times_formatted, 'delay': target, 'predicted delay': y_pred } fname = '{}/delays_{}.csv'.format(options.vis_path, station) io.write_csv(delay_data, fname, fname) # Draw visualisation if predictor.y_pred_bin_proba is not None: fname = '{}/timeseries_proba_{}'.format(options.vis_path, station) proba = predictor.y_pred_bin_proba[:, 1] viz.plot_delay(times, target, None, 'Delay for station {}'.format(stationName), fname, all_proba=proba, proba_mode='same', color_threshold=options.class_limit) #else: fname = '{}/timeseries_regression_{}'.format(options.vis_path, station) viz.plot_delay(times, target, y_pred, 'Delay for station {}'.format(stationName), fname, all_proba=None) fname = '{}/scatter_all_stations.png'.format(options.vis_path) viz.scatter_predictions(times, target, y_pred, savepath=options.vis_path, filename='scatter_{}'.format(station)) # Save all station related results to csv and upload them to bucket fname = '{}/station_rmse.csv'.format(options.vis_path) io.dict_to_csv(station_rmse, fname, fname) fname = '{}/station_mae.csv'.format(options.vis_path) io.dict_to_csv(station_mae, fname, fname) fname = '{}/station_r2.csv'.format(options.vis_path) io.dict_to_csv(station_r2, fname, fname) fname = '{}/station_skill_rmse.csv'.format(options.vis_path) io.dict_to_csv(station_skill, fname, fname) # Create timeseries of avg actual delay and predicted delay all_times = sorted(list(all_times)) for t, l in avg_delay.items(): avg_delay[t] = sum(l) / len(l) for t, l in avg_pred_delay.items(): avg_pred_delay[t] = sum(l) / len(l) for t, l in avg_proba.items(): avg_proba[t] = sum(l) / len(l) avg_delay = list( OrderedDict(sorted(avg_delay.items(), key=lambda t: t[0])).values()) avg_pred_delay = list( OrderedDict(sorted(avg_pred_delay.items(), key=lambda t: t[0])).values()) avg_proba = list( OrderedDict(sorted(avg_proba.items(), key=lambda t: t[0])).values()) # Calculate average over all times and stations, first for all months separately, then for whole time range splits = viz._split_to_parts(list(times), [avg_delay, avg_pred_delay], 2592000) for i in range(0, len(splits)): times_, avg_delay_, avg_pred_delay_ = splits[i] try: rmse = math.sqrt( metrics.mean_squared_error(avg_delay_, avg_pred_delay_)) mae = metrics.mean_absolute_error(avg_delay_, avg_pred_delay_) r2 = metrics.r2_score(avg_delay_, avg_pred_delay_) rmse_stat = math.sqrt( metrics.mean_squared_error( avg_delay_, np.full_like(avg_delay_, mean_delay))) skill = 1 - rmse / rmse_stat except ValueError: logging.warning('Zero samples in some class') continue logging.info('Month: {}'.format(i + 1)) logging.info( 'RMSE of average delay over all stations: {:.4f}'.format(rmse)) logging.info( 'MAE of average delay over all stations: {:.4f}'.format(mae)) logging.info( 'R2 score of average delay over all stations: {:.4f}'.format(r2)) logging.info( 'Skill score (RMSE) of average delay over all stations: {:.4f}'. format(skill)) # Write average data into file avg_errors = { 'rmse': rmse, 'mae': mae, 'r2': r2, 'skill': skill, 'nro_of_samples': len(avg_delay) } fname = '{}/avg_erros_{}.csv'.format(options.vis_path, i) io.dict_to_csv(avg_errors, fname, fname) rmse = math.sqrt(metrics.mean_squared_error(avg_delay, avg_pred_delay)) #rmse_mean = np.mean(list(station_rmse.values())) mae = metrics.mean_absolute_error(avg_delay, avg_pred_delay) #mae_mean = np.mean(list(station_mae.values())) r2 = metrics.r2_score(avg_delay, avg_pred_delay) rmse_stat = math.sqrt( metrics.mean_squared_error(avg_delay, np.full_like(avg_delay, mean_delay))) skill = 1 - rmse / rmse_stat #skill_mean = 1 - rmse_mean/rmse_stat logging.info('All periods:') logging.info( 'RMSE of average delay over all stations: {:.4f}'.format(rmse)) #logging.info('Average RMSE of all station RMSEs: {:.4f}'.format(rmse_mean)) logging.info('MAE of average delay over all stations: {:.4f}'.format(mae)) #logging.info('Average MAE of all station MAEs: {:.4f}'.format(mae_mean)) logging.info( 'R2 score of average delay over all stations: {:.4f}'.format(r2)) logging.info( 'Skill score (RMSE) of average delay over all stations: {:.4f}'.format( skill)) #logging.info('Skill score (avg RMSE) of all stations: {:.4f}'.format(skill_mean)) # Write average data into file avg_errors = { 'rmse': rmse, 'mae': mae, 'r2': r2, #'rmse_mean': rmse_mean, #'mae_mean': mae_mean, 'skill': skill, #'skill_mean': skill_mean, 'nro_of_samples': len(avg_delay) } fname = '{}/avg_erros.csv'.format(options.vis_path) io.dict_to_csv(avg_errors, fname, fname) # Create timeseries of average delay and predicted delays over all stations all_times_formatted = [t.strftime('%Y-%m-%dT%H:%M:%S') for t in all_times] delay_data = { 'times': all_times_formatted, 'delay': avg_delay, 'predicted delay': avg_pred_delay } # write csv fname = '{}/avg_delays_all_stations.csv'.format(options.vis_path) io.write_csv(delay_data, fname, fname) # visualise if not avg_proba: proba = None else: proba = avg_proba fname = '{}/timeseries_avg_all_stations.png'.format(options.vis_path) if predictor.y_pred_bin is not None: viz.plot_delay(all_times, avg_delay, None, 'Average delay for all station', fname, all_proba=proba, proba_mode='same', color_threshold=options.class_limit) else: viz.plot_delay(all_times, avg_delay, avg_pred_delay, 'Average delay for all station', fname) fname = '{}/scatter_all_stations.png'.format(options.vis_path) viz.scatter_predictions(all_times, avg_delay, avg_pred_delay, savepath=options.vis_path, filename='scatter_all_stations') # Binary classification metrics if predictor.y_pred_bin is not None: all_data.sort_values(by=['time'], inplace=True) times = all_data.loc[:, 'time'].values try: target = all_data.loc[:, options.label_params].reset_index( drop=True).values.ravel() y_pred, y_pred_bin, y_pred_bin_proba = predictor.pred( times, all_data) # Drop first times which LSTM are not able to predict times = times[(len(all_data) - len(y_pred)):] splits = viz._split_to_parts(list(times), [ target, y_pred, predictor.y_pred_bin, predictor.y_pred_bin_proba ], 2592000) for i in range(0, len(splits)): #times_, target_, y_pred_bin_, y_pred_bin_proba_ = splits[i] times_, target_, y_pred_, y_pred_bin_, y_pred_bin_proba_ = splits[ i] viz.classification_perf_metrics(y_pred_bin_proba_, y_pred_bin_, target_, options, times_, 'all') except (PredictionError, ModelError) as e: logging.error(e) pass