def run_test(test_data2d, test_data_normalized2d, fwbw_conv_lstm_net, scalers, results_summary): err, r2_score, rmse = [], [], [] err_ims, r2_score_ims, rmse_ims = [], [], [] for i in range(Config.FWBW_CONV_LSTM_TESTING_TIME): print('|--- Run time {}'.format(i)) test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days( test_data2d, test_data_normalized2d) # test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d) init_data_normalize = np.reshape( init_data_normalize, newshape=(init_data_normalize.shape[0], Config.FWBW_CONV_LSTM_WIDE, Config.FWBW_CONV_LSTM_HIGH)) test_data_normalize = np.reshape( test_data_normalize, newshape=(test_data_normalize.shape[0], Config.FWBW_CONV_LSTM_WIDE, Config.FWBW_CONV_LSTM_HIGH)) measured_matrix_ims2d = np.zeros( (test_data.shape[0] - Config.FWBW_CONV_LSTM_IMS_STEP + 1, Config.FWBW_CONV_LSTM_WIDE * Config.FWBW_CONV_LSTM_HIGH)) pred_tm, measured_matrix, ims_tm = predict_fwbw_conv_lstm( initial_data=init_data_normalize, test_data=test_data_normalize, model=fwbw_conv_lstm_net.model) pred_tm2d = np.reshape(pred_tm, newshape=(pred_tm.shape[0], pred_tm.shape[1] * pred_tm.shape[2])) measured_matrix2d = np.reshape( measured_matrix, newshape=(measured_matrix.shape[0], measured_matrix.shape[1] * measured_matrix.shape[2])) pred_tm_invert2d = scalers.inverse_transform(pred_tm2d) if np.any(np.isinf(pred_tm_invert2d)): raise ValueError('Value is infinity!') elif np.any(np.isnan(pred_tm_invert2d)): raise ValueError('Value is NaN!') err.append( error_ratio(y_true=test_data, y_pred=pred_tm_invert2d, measured_matrix=measured_matrix2d)) r2_score.append( calculate_r2_score(y_true=test_data, y_pred=pred_tm_invert2d)) rmse.append( calculate_rmse(y_true=test_data / 1000000, y_pred=pred_tm_invert2d / 1000000)) if Config.FWBW_CONV_LSTM_IMS: # Calculate error for multistep-ahead-prediction ims_tm2d = np.reshape(np.copy(ims_tm), newshape=(ims_tm.shape[0], ims_tm.shape[1] * ims_tm.shape[2])) ims_tm_invert2d = scalers.inverse_transform(ims_tm2d) ims_ytrue2d = ims_tm_test_data(test_data=test_data) err_ims.append( error_ratio(y_pred=ims_tm_invert2d, y_true=ims_ytrue2d, measured_matrix=measured_matrix_ims2d)) r2_score_ims.append( calculate_r2_score(y_true=ims_ytrue2d, y_pred=ims_tm_invert2d)) rmse_ims.append( calculate_rmse(y_true=ims_ytrue2d / 1000000, y_pred=ims_tm_invert2d / 1000000)) else: err_ims.append(0) r2_score_ims.append(0) rmse_ims.append(0) print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims') print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format( err[i], rmse[i], r2_score[i], err_ims[i], rmse_ims[i], r2_score_ims[i])) results_summary['No.'] = range(Config.FWBW_CONV_LSTM_TESTING_TIME) results_summary['err'] = err results_summary['r2'] = r2_score results_summary['rmse'] = rmse results_summary['err_ims'] = err_ims results_summary['r2_ims'] = r2_score_ims results_summary['rmse_ims'] = rmse_ims print('Test: {}-{}-{}-{}'.format(Config.DATA_NAME, Config.ALG, Config.TAG, Config.SCALER)) print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format( np.mean(np.array(err)), np.mean(np.array(rmse)), np.mean(np.array(r2_score)))) return results_summary
def run_test(test_data2d, test_data_normalized2d, lstm_net, scalers, results_summary): err, r2_score, rmse = [], [], [] err_ims, r2_score_ims, rmse_ims = [], [], [] for i in range(Config.RES_LSTM_TESTING_TIME): print('|--- Running time: {}'.format(i)) # test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d) test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days( test_data2d, test_data_normalized2d) ims_test_data = ims_tm_test_data(test_data=test_data) measured_matrix_ims = np.zeros(shape=ims_test_data.shape) pred_tm2d, measured_matrix2d, ims_tm2d = predict_lstm_nn( init_data=init_data_normalize, test_data=test_data_normalize, model=lstm_net.model) pred_tm_invert2d = scalers.inverse_transform(pred_tm2d) err.append( error_ratio(y_true=test_data, y_pred=pred_tm_invert2d, measured_matrix=measured_matrix2d)) r2_score.append( calculate_r2_score(y_true=test_data, y_pred=pred_tm_invert2d)) rmse.append( calculate_rmse(y_true=test_data / 1000000, y_pred=pred_tm_invert2d / 1000000)) if Config.RES_LSTM_IMS: ims_tm_invert2d = scalers.inverse_transform(ims_tm2d) err_ims.append( error_ratio(y_pred=ims_tm_invert2d, y_true=ims_test_data, measured_matrix=measured_matrix_ims)) r2_score_ims.append( calculate_r2_score(y_true=ims_test_data, y_pred=ims_tm_invert2d)) rmse_ims.append( calculate_rmse(y_true=ims_test_data / 1000000, y_pred=ims_tm_invert2d / 1000000)) else: err_ims.append(0) r2_score_ims.append(0) rmse_ims.append(0) print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims') print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format( err[i], rmse[i], r2_score[i], err_ims[i], rmse_ims[i], r2_score_ims[i])) results_summary['No.'] = range(Config.RES_LSTM_TESTING_TIME) results_summary['err'] = err results_summary['r2'] = r2_score results_summary['rmse'] = rmse results_summary['err_ims'] = err_ims results_summary['r2_ims'] = r2_score_ims results_summary['rmse_ims'] = rmse_ims print('Test: {}-{}-{}-{}'.format(Config.DATA_NAME, Config.ALG, Config.TAG, Config.SCALER)) print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format( np.mean(np.array(err)), np.mean(np.array(rmse)), np.mean(np.array(r2_score)))) return results_summary
def run_test(test_data2d, test_data_normalized2d, fwbw_net, scalers, results_summary): mape, r2_score, rmse = [], [], [] mape_ims, r2_score_ims, rmse_ims = [], [], [] # per_gain = [] for i in range(config['test']['run_times']): print('|--- Run time {}'.format(i)) # test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d) test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days( test_data2d, test_data_normalized2d) ims_test_data = ims_tm_test_data(test_data=test_data) measured_matrix_ims = np.zeros(shape=ims_test_data.shape) pred_tm2d, measured_matrix2d, ims_tm2d, predicted_tm2d = predict_fwbw_lstm_v2( initial_data=init_data_normalize, test_data=test_data_normalize, model=fwbw_net.model) pred_tm_invert2d = scalers.inverse_transform(pred_tm2d) predicted_tm_invert2d = scalers.inverse_transform(predicted_tm2d) # pred_tm_wo_invert2d = scalers.inverse_transform(pred_tm2d_wo) if np.any(np.isinf(pred_tm_invert2d)): raise ValueError('Value is infinity!') elif np.any(np.isnan(pred_tm_invert2d)): raise ValueError('Value is NaN!') if np.any(np.isinf(predicted_tm_invert2d)): raise ValueError('Value is infinity!') elif np.any(np.isnan(predicted_tm_invert2d)): raise ValueError('Value is NaN!') r2_score.append( calculate_r2_score(y_true=test_data, y_pred=predicted_tm_invert2d)) rmse.append( calculate_rmse(y_true=test_data, y_pred=predicted_tm_invert2d)) mape.append( calculate_mape(y_true=test_data, y_pred=predicted_tm_invert2d)) if config['model']['horizon']: # Calculate error for multistep-ahead-prediction ims_tm_invert2d = scalers.inverse_transform(ims_tm2d) # err_ims.append(error_ratio(y_pred=ims_tm_invert2d, # y_true=ims_test_data, # measured_matrix=measured_matrix_ims)) mape_ims.append( calculate_mape(y_true=ims_test_data, y_pred=ims_tm_invert2d)) r2_score_ims.append( calculate_r2_score(y_true=ims_test_data, y_pred=ims_tm_invert2d)) rmse_ims.append( calculate_rmse(y_true=ims_test_data, y_pred=ims_tm_invert2d)) else: mape_ims.append(0) r2_score_ims.append(0) rmse_ims.append(0) print('Result: mape\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims') print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format( mape[i], rmse[i], r2_score[i], mape_ims[i], rmse_ims[i], r2_score_ims[i])) results_summary['No.'] = range(config['test']['run_times']) results_summary['mape'] = mape results_summary['r2'] = r2_score results_summary['rmse'] = rmse results_summary['mape_ims'] = mape_ims results_summary['r2_ims'] = r2_score_ims results_summary['rmse_ims'] = rmse_ims print('Test: {}-{}-{}-{}-{}'.format(config['data']['data_name'], config['alg'], Config.TAG, Config.SCALER, config['test']['flow_selection'])) print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format( np.mean(np.array(mape)), np.mean(np.array(rmse)), np.mean(np.array(r2_score)))) return results_summary
def run_test(test_data2d, test_data_normalized2d, fwbw_net, scalers, results_summary): err, r2_score, rmse = [], [], [] err_ims, r2_score_ims, rmse_ims = [], [], [] # per_gain = [] for i in range(Config.RES_FWBW_LSTM_TESTING_TIME): print('|--- Run time {}'.format(i)) # test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d) test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days( test_data2d, test_data_normalized2d) ims_test_data = ims_tm_test_data(test_data=test_data) measured_matrix_ims = np.zeros(shape=ims_test_data.shape) pred_tm2d, measured_matrix2d, ims_tm2d = predict_fwbw_lstm_v2( initial_data=init_data_normalize, test_data=test_data_normalize, model=fwbw_net.model) pred_tm_invert2d = scalers.inverse_transform(pred_tm2d) # pred_tm_wo_invert2d = scalers.inverse_transform(pred_tm2d_wo) if np.any(np.isinf(pred_tm_invert2d)): raise ValueError('Value is infinity!') elif np.any(np.isnan(pred_tm_invert2d)): raise ValueError('Value is NaN!') err.append( error_ratio(y_true=test_data, y_pred=pred_tm_invert2d, measured_matrix=measured_matrix2d)) r2_score.append( calculate_r2_score(y_true=test_data, y_pred=pred_tm_invert2d)) rmse.append( calculate_rmse(y_true=test_data / 1000000, y_pred=pred_tm_invert2d / 1000000)) # err_wo = error_ratio(y_true=test_data, y_pred=pred_tm_wo_invert2d, measured_matrix=measured_matrix2d) # r2_score_wo = calculate_r2_score(y_true=test_data, y_pred=pred_tm_wo_invert2d) # rmse_wo = calculate_rmse(y_true=test_data / 1000000, y_pred=pred_tm_wo_invert2d / 1000000) if Config.RES_FWBW_LSTM_IMS: # Calculate error for multistep-ahead-prediction ims_tm_invert2d = scalers.inverse_transform(ims_tm2d) err_ims.append( error_ratio(y_pred=ims_tm_invert2d, y_true=ims_test_data, measured_matrix=measured_matrix_ims)) r2_score_ims.append( calculate_r2_score(y_true=ims_test_data, y_pred=ims_tm_invert2d)) rmse_ims.append( calculate_rmse(y_true=ims_test_data / 1000000, y_pred=ims_tm_invert2d / 1000000)) else: err_ims.append(0) r2_score_ims.append(0) rmse_ims.append(0) print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims') print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format( err[i], rmse[i], r2_score[i], err_ims[i], rmse_ims[i], r2_score_ims[i])) # print('Result without data correction: mape \t err\trmse\tr2') # print(' {}\t{}\t{}\t{}'.format(mape_wo, err_wo, rmse_wo, r2_score_wo)) # # if err[i] < err_wo: # per_gain.append(np.abs(err[i] - err_wo) * 100.0 / err_wo) # print('|-----> Performance gain: {}'.format(np.abs(err[i] - err_wo) * 100.0 / err_wo)) # else: # per_gain.append(-np.abs(err[i] - err_wo) * 100.0 / err[i]) # print('|-----> Performance gain: {}'.format(-np.abs(err[i] - err_wo) * 100.0 / err[i])) results_summary['No.'] = range(Config.RES_FWBW_LSTM_TESTING_TIME) results_summary['err'] = err results_summary['r2'] = r2_score results_summary['rmse'] = rmse results_summary['err_ims'] = err_ims results_summary['r2_ims'] = r2_score_ims results_summary['rmse_ims'] = rmse_ims print('Test: {}-{}-{}-{}'.format(Config.DATA_NAME, Config.ALG, Config.TAG, Config.SCALER)) print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format( np.mean(np.array(err)), np.mean(np.array(rmse)), np.mean(np.array(r2_score)))) # print('Avg_per_gain: {} - Confidence: {}'.format(np.mean(np.array(per_gain)), # calculate_confident_interval(per_gain))) return results_summary
def run_test(test_data2d, test_data_normalized2d, init_data2d, net, params, scalers): alg_name = Config.ALG tag = Config.TAG data_name = Config.DATA_NAME results_summary = pd.DataFrame(index=range(Config.FWBW_CONVLSTM_TESTING_TIME), columns=['No.', 'err', 'r2', 'rmse', 'err_ims', 'r2_ims', 'rmse_ims']) err, r2_score, rmse = [], [], [] err_ims, r2_score_ims, rmse_ims = [], [], [] measured_matrix_ims2d = np.zeros((test_data2d.shape[0] - Config.FWBW_CONVLSTM_IMS_STEP + 1, Config.FWBW_CONVLSTM_WIDE * Config.FWBW_CONVLSTM_HIGH)) # if not os.path.isfile(Config.RESULTS_PATH + 'ground_true_{}.npy'.format(data_name)): # np.save(Config.RESULTS_PATH + 'ground_true_{}.npy'.format(data_name), # test_data2d) # if not os.path.isfile(Config.RESULTS_PATH + 'ground_true_scaled_{}_{}.npy'.format(data_name, Config.SCALER)): # np.save(Config.RESULTS_PATH + 'ground_true_scaled_{}_{}.npy'.format(data_name, Config.SCALER), # test_data_normalized2d) if not os.path.exists(Config.RESULTS_PATH + '{}-{}-{}-{}/'.format(data_name, alg_name, tag, Config.SCALER)): os.makedirs(Config.RESULTS_PATH + '{}-{}-{}-{}/'.format(data_name, alg_name, tag, Config.SCALER)) for i in range(Config.FWBW_CONVLSTM_TESTING_TIME): print('|--- Run time {}'.format(i)) init_data = np.reshape(init_data2d, newshape=(init_data2d.shape[0], Config.FWBW_CONVLSTM_WIDE, Config.FWBW_CONVLSTM_HIGH)) test_data_normalized = np.reshape(test_data_normalized2d, newshape=(test_data_normalized2d.shape[0], Config.FWBW_CONVLSTM_WIDE, Config.FWBW_CONVLSTM_HIGH)) pred_tm, measured_matrix, ims_tm = predict_fwbw_convlstm(initial_data=init_data, test_data=test_data_normalized, model=net.model) pred_tm2d = np.reshape(np.copy(pred_tm), newshape=(pred_tm.shape[0], pred_tm.shape[1] * pred_tm.shape[2])) measured_matrix2d = np.reshape(np.copy(measured_matrix), newshape=(measured_matrix.shape[0], measured_matrix.shape[1] * measured_matrix.shape[2])) # np.save(Config.RESULTS_PATH + '{}-{}-{}-{}/pred_scaled-{}.npy'.format(data_name, alg_name, tag, # Config.SCALER, i), # pred_tm2d) pred_tm_invert2d = scalers.inverse_transform(pred_tm2d) if np.any(np.isinf(pred_tm_invert2d)): raise ValueError('Value is infinity!') elif np.any(np.isnan(pred_tm_invert2d)): raise ValueError('Value is NaN!') err.append(error_ratio(y_true=test_data2d, y_pred=pred_tm_invert2d, measured_matrix=measured_matrix2d)) r2_score.append(calculate_r2_score(y_true=test_data2d, y_pred=pred_tm_invert2d)) rmse.append(calculate_rmse(y_true=test_data2d / 1000000, y_pred=pred_tm_invert2d / 1000000)) if Config.FWBW_IMS: # Calculate error for multistep-ahead-prediction ims_tm2d = np.reshape(np.copy(ims_tm), newshape=(ims_tm.shape[0], ims_tm.shape[1] * ims_tm.shape[2])) ims_tm_invert2d = scalers.inverse_transform(ims_tm2d) ims_ytrue2d = ims_tm_test_data(test_data=test_data2d) err_ims.append(error_ratio(y_pred=ims_tm_invert2d, y_true=ims_ytrue2d, measured_matrix=measured_matrix_ims2d)) r2_score_ims.append(calculate_r2_score(y_true=ims_ytrue2d, y_pred=ims_tm_invert2d)) rmse_ims.append(calculate_rmse(y_true=ims_ytrue2d / 1000000, y_pred=ims_tm_invert2d / 1000000)) else: err_ims.append(0) r2_score_ims.append(0) rmse_ims.append(0) print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims') print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format(err[i], rmse[i], r2_score[i], err_ims[i], rmse_ims[i], r2_score_ims[i])) # np.save(Config.RESULTS_PATH + '{}-{}-{}-{}/pred-{}.npy'.format(data_name, alg_name, tag, # Config.SCALER, i), # pred_tm_invert2d) # np.save(Config.RESULTS_PATH + '{}-{}-{}-{}/measure-{}.npy'.format(data_name, alg_name, tag, # Config.SCALER, i), # measured_matrix2d) results_summary['No.'] = range(Config.FWBW_CONVLSTM_TESTING_TIME) results_summary['err'] = err results_summary['r2'] = r2_score results_summary['rmse'] = rmse results_summary['err_ims'] = err_ims results_summary['r2_ims'] = r2_score_ims results_summary['rmse_ims'] = rmse_ims results_summary.to_csv(Config.RESULTS_PATH + '{}-{}-{}-{}/results.csv'.format(data_name, alg_name, tag, Config.SCALER), index=False) print('Test: {}-{}-{}-{}'.format(data_name, alg_name, tag, Config.SCALER)) print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format(np.mean(np.array(err)), np.mean(np.array(rmse)), np.mean(np.array(r2_score)))) return