def run_test(test_data2d, test_data_normalized2d, fwbw_conv_lstm_net, scalers,
results_summary):
err, r2_score, rmse = [], [], []
err_ims, r2_score_ims, rmse_ims = [], [], []
for i in range(Config.FWBW_CONV_LSTM_TESTING_TIME):
print('|--- Run time {}'.format(i))
test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days(
test_data2d, test_data_normalized2d)
# test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d)
init_data_normalize = np.reshape(
init_data_normalize,
newshape=(init_data_normalize.shape[0], Config.FWBW_CONV_LSTM_WIDE,
Config.FWBW_CONV_LSTM_HIGH))
test_data_normalize = np.reshape(
test_data_normalize,
newshape=(test_data_normalize.shape[0], Config.FWBW_CONV_LSTM_WIDE,
Config.FWBW_CONV_LSTM_HIGH))
measured_matrix_ims2d = np.zeros(
(test_data.shape[0] - Config.FWBW_CONV_LSTM_IMS_STEP + 1,
Config.FWBW_CONV_LSTM_WIDE * Config.FWBW_CONV_LSTM_HIGH))
pred_tm, measured_matrix, ims_tm = predict_fwbw_conv_lstm(
initial_data=init_data_normalize,
test_data=test_data_normalize,
model=fwbw_conv_lstm_net.model)
pred_tm2d = np.reshape(pred_tm,
newshape=(pred_tm.shape[0],
pred_tm.shape[1] * pred_tm.shape[2]))
measured_matrix2d = np.reshape(
measured_matrix,
newshape=(measured_matrix.shape[0],
measured_matrix.shape[1] * measured_matrix.shape[2]))
pred_tm_invert2d = scalers.inverse_transform(pred_tm2d)
if np.any(np.isinf(pred_tm_invert2d)):
raise ValueError('Value is infinity!')
elif np.any(np.isnan(pred_tm_invert2d)):
raise ValueError('Value is NaN!')
err.append(
error_ratio(y_true=test_data,
y_pred=pred_tm_invert2d,
measured_matrix=measured_matrix2d))
r2_score.append(
calculate_r2_score(y_true=test_data, y_pred=pred_tm_invert2d))
rmse.append(
calculate_rmse(y_true=test_data / 1000000,
y_pred=pred_tm_invert2d / 1000000))
if Config.FWBW_CONV_LSTM_IMS:
# Calculate error for multistep-ahead-prediction
ims_tm2d = np.reshape(np.copy(ims_tm),
newshape=(ims_tm.shape[0],
ims_tm.shape[1] * ims_tm.shape[2]))
ims_tm_invert2d = scalers.inverse_transform(ims_tm2d)
ims_ytrue2d = ims_tm_test_data(test_data=test_data)
err_ims.append(
error_ratio(y_pred=ims_tm_invert2d,
y_true=ims_ytrue2d,
measured_matrix=measured_matrix_ims2d))
r2_score_ims.append(
calculate_r2_score(y_true=ims_ytrue2d, y_pred=ims_tm_invert2d))
rmse_ims.append(
calculate_rmse(y_true=ims_ytrue2d / 1000000,
y_pred=ims_tm_invert2d / 1000000))
else:
err_ims.append(0)
r2_score_ims.append(0)
rmse_ims.append(0)
print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims')
print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format(
err[i], rmse[i], r2_score[i], err_ims[i], rmse_ims[i],
r2_score_ims[i]))
results_summary['No.'] = range(Config.FWBW_CONV_LSTM_TESTING_TIME)
results_summary['err'] = err
results_summary['r2'] = r2_score
results_summary['rmse'] = rmse
results_summary['err_ims'] = err_ims
results_summary['r2_ims'] = r2_score_ims
results_summary['rmse_ims'] = rmse_ims
print('Test: {}-{}-{}-{}'.format(Config.DATA_NAME, Config.ALG, Config.TAG,
Config.SCALER))
print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format(
np.mean(np.array(err)), np.mean(np.array(rmse)),
np.mean(np.array(r2_score))))
return results_summary
def run_test(test_data2d, test_data_normalized2d, lstm_net, scalers,
results_summary):
err, r2_score, rmse = [], [], []
err_ims, r2_score_ims, rmse_ims = [], [], []
for i in range(Config.RES_LSTM_TESTING_TIME):
print('|--- Running time: {}'.format(i))
# test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d)
test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days(
test_data2d, test_data_normalized2d)
ims_test_data = ims_tm_test_data(test_data=test_data)
measured_matrix_ims = np.zeros(shape=ims_test_data.shape)
pred_tm2d, measured_matrix2d, ims_tm2d = predict_lstm_nn(
init_data=init_data_normalize,
test_data=test_data_normalize,
model=lstm_net.model)
pred_tm_invert2d = scalers.inverse_transform(pred_tm2d)
err.append(
error_ratio(y_true=test_data,
y_pred=pred_tm_invert2d,
measured_matrix=measured_matrix2d))
r2_score.append(
calculate_r2_score(y_true=test_data, y_pred=pred_tm_invert2d))
rmse.append(
calculate_rmse(y_true=test_data / 1000000,
y_pred=pred_tm_invert2d / 1000000))
if Config.RES_LSTM_IMS:
ims_tm_invert2d = scalers.inverse_transform(ims_tm2d)
err_ims.append(
error_ratio(y_pred=ims_tm_invert2d,
y_true=ims_test_data,
measured_matrix=measured_matrix_ims))
r2_score_ims.append(
calculate_r2_score(y_true=ims_test_data,
y_pred=ims_tm_invert2d))
rmse_ims.append(
calculate_rmse(y_true=ims_test_data / 1000000,
y_pred=ims_tm_invert2d / 1000000))
else:
err_ims.append(0)
r2_score_ims.append(0)
rmse_ims.append(0)
print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims')
print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format(
err[i], rmse[i], r2_score[i], err_ims[i], rmse_ims[i],
r2_score_ims[i]))
results_summary['No.'] = range(Config.RES_LSTM_TESTING_TIME)
results_summary['err'] = err
results_summary['r2'] = r2_score
results_summary['rmse'] = rmse
results_summary['err_ims'] = err_ims
results_summary['r2_ims'] = r2_score_ims
results_summary['rmse_ims'] = rmse_ims
print('Test: {}-{}-{}-{}'.format(Config.DATA_NAME, Config.ALG, Config.TAG,
Config.SCALER))
print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format(
np.mean(np.array(err)), np.mean(np.array(rmse)),
np.mean(np.array(r2_score))))
return results_summary
def run_test(test_data2d, test_data_normalized2d, fwbw_net, scalers,
results_summary):
mape, r2_score, rmse = [], [], []
mape_ims, r2_score_ims, rmse_ims = [], [], []
# per_gain = []
for i in range(config['test']['run_times']):
print('|--- Run time {}'.format(i))
# test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d)
test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days(
test_data2d, test_data_normalized2d)
ims_test_data = ims_tm_test_data(test_data=test_data)
measured_matrix_ims = np.zeros(shape=ims_test_data.shape)
pred_tm2d, measured_matrix2d, ims_tm2d, predicted_tm2d = predict_fwbw_lstm_v2(
initial_data=init_data_normalize,
test_data=test_data_normalize,
model=fwbw_net.model)
pred_tm_invert2d = scalers.inverse_transform(pred_tm2d)
predicted_tm_invert2d = scalers.inverse_transform(predicted_tm2d)
# pred_tm_wo_invert2d = scalers.inverse_transform(pred_tm2d_wo)
if np.any(np.isinf(pred_tm_invert2d)):
raise ValueError('Value is infinity!')
elif np.any(np.isnan(pred_tm_invert2d)):
raise ValueError('Value is NaN!')
if np.any(np.isinf(predicted_tm_invert2d)):
raise ValueError('Value is infinity!')
elif np.any(np.isnan(predicted_tm_invert2d)):
raise ValueError('Value is NaN!')
r2_score.append(
calculate_r2_score(y_true=test_data, y_pred=predicted_tm_invert2d))
rmse.append(
calculate_rmse(y_true=test_data, y_pred=predicted_tm_invert2d))
mape.append(
calculate_mape(y_true=test_data, y_pred=predicted_tm_invert2d))
if config['model']['horizon']:
# Calculate error for multistep-ahead-prediction
ims_tm_invert2d = scalers.inverse_transform(ims_tm2d)
# err_ims.append(error_ratio(y_pred=ims_tm_invert2d,
# y_true=ims_test_data,
# measured_matrix=measured_matrix_ims))
mape_ims.append(
calculate_mape(y_true=ims_test_data, y_pred=ims_tm_invert2d))
r2_score_ims.append(
calculate_r2_score(y_true=ims_test_data,
y_pred=ims_tm_invert2d))
rmse_ims.append(
calculate_rmse(y_true=ims_test_data, y_pred=ims_tm_invert2d))
else:
mape_ims.append(0)
r2_score_ims.append(0)
rmse_ims.append(0)
print('Result: mape\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims')
print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format(
mape[i], rmse[i], r2_score[i], mape_ims[i], rmse_ims[i],
r2_score_ims[i]))
results_summary['No.'] = range(config['test']['run_times'])
results_summary['mape'] = mape
results_summary['r2'] = r2_score
results_summary['rmse'] = rmse
results_summary['mape_ims'] = mape_ims
results_summary['r2_ims'] = r2_score_ims
results_summary['rmse_ims'] = rmse_ims
print('Test: {}-{}-{}-{}-{}'.format(config['data']['data_name'],
config['alg'], Config.TAG,
Config.SCALER,
config['test']['flow_selection']))
print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format(
np.mean(np.array(mape)), np.mean(np.array(rmse)),
np.mean(np.array(r2_score))))
return results_summary
def run_test(test_data2d, test_data_normalized2d, fwbw_net, scalers,
results_summary):
err, r2_score, rmse = [], [], []
err_ims, r2_score_ims, rmse_ims = [], [], []
# per_gain = []
for i in range(Config.RES_FWBW_LSTM_TESTING_TIME):
print('|--- Run time {}'.format(i))
# test_data_normalize, init_data_normalize, test_data = prepare_test_set(test_data2d, test_data_normalized2d)
test_data_normalize, init_data_normalize, test_data = prepare_test_set_last_5days(
test_data2d, test_data_normalized2d)
ims_test_data = ims_tm_test_data(test_data=test_data)
measured_matrix_ims = np.zeros(shape=ims_test_data.shape)
pred_tm2d, measured_matrix2d, ims_tm2d = predict_fwbw_lstm_v2(
initial_data=init_data_normalize,
test_data=test_data_normalize,
model=fwbw_net.model)
pred_tm_invert2d = scalers.inverse_transform(pred_tm2d)
# pred_tm_wo_invert2d = scalers.inverse_transform(pred_tm2d_wo)
if np.any(np.isinf(pred_tm_invert2d)):
raise ValueError('Value is infinity!')
elif np.any(np.isnan(pred_tm_invert2d)):
raise ValueError('Value is NaN!')
err.append(
error_ratio(y_true=test_data,
y_pred=pred_tm_invert2d,
measured_matrix=measured_matrix2d))
r2_score.append(
calculate_r2_score(y_true=test_data, y_pred=pred_tm_invert2d))
rmse.append(
calculate_rmse(y_true=test_data / 1000000,
y_pred=pred_tm_invert2d / 1000000))
# err_wo = error_ratio(y_true=test_data, y_pred=pred_tm_wo_invert2d, measured_matrix=measured_matrix2d)
# r2_score_wo = calculate_r2_score(y_true=test_data, y_pred=pred_tm_wo_invert2d)
# rmse_wo = calculate_rmse(y_true=test_data / 1000000, y_pred=pred_tm_wo_invert2d / 1000000)
if Config.RES_FWBW_LSTM_IMS:
# Calculate error for multistep-ahead-prediction
ims_tm_invert2d = scalers.inverse_transform(ims_tm2d)
err_ims.append(
error_ratio(y_pred=ims_tm_invert2d,
y_true=ims_test_data,
measured_matrix=measured_matrix_ims))
r2_score_ims.append(
calculate_r2_score(y_true=ims_test_data,
y_pred=ims_tm_invert2d))
rmse_ims.append(
calculate_rmse(y_true=ims_test_data / 1000000,
y_pred=ims_tm_invert2d / 1000000))
else:
err_ims.append(0)
r2_score_ims.append(0)
rmse_ims.append(0)
print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims')
print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format(
err[i], rmse[i], r2_score[i], err_ims[i], rmse_ims[i],
r2_score_ims[i]))
# print('Result without data correction: mape \t err\trmse\tr2')
# print(' {}\t{}\t{}\t{}'.format(mape_wo, err_wo, rmse_wo, r2_score_wo))
#
# if err[i] < err_wo:
# per_gain.append(np.abs(err[i] - err_wo) * 100.0 / err_wo)
# print('|-----> Performance gain: {}'.format(np.abs(err[i] - err_wo) * 100.0 / err_wo))
# else:
# per_gain.append(-np.abs(err[i] - err_wo) * 100.0 / err[i])
# print('|-----> Performance gain: {}'.format(-np.abs(err[i] - err_wo) * 100.0 / err[i]))
results_summary['No.'] = range(Config.RES_FWBW_LSTM_TESTING_TIME)
results_summary['err'] = err
results_summary['r2'] = r2_score
results_summary['rmse'] = rmse
results_summary['err_ims'] = err_ims
results_summary['r2_ims'] = r2_score_ims
results_summary['rmse_ims'] = rmse_ims
print('Test: {}-{}-{}-{}'.format(Config.DATA_NAME, Config.ALG, Config.TAG,
Config.SCALER))
print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format(
np.mean(np.array(err)), np.mean(np.array(rmse)),
np.mean(np.array(r2_score))))
# print('Avg_per_gain: {} - Confidence: {}'.format(np.mean(np.array(per_gain)),
# calculate_confident_interval(per_gain)))
return results_summary
def run_test(test_data2d, test_data_normalized2d, init_data2d, net, params, scalers):
alg_name = Config.ALG
tag = Config.TAG
data_name = Config.DATA_NAME
results_summary = pd.DataFrame(index=range(Config.FWBW_CONVLSTM_TESTING_TIME),
columns=['No.', 'err', 'r2', 'rmse', 'err_ims', 'r2_ims', 'rmse_ims'])
err, r2_score, rmse = [], [], []
err_ims, r2_score_ims, rmse_ims = [], [], []
measured_matrix_ims2d = np.zeros((test_data2d.shape[0] - Config.FWBW_CONVLSTM_IMS_STEP + 1,
Config.FWBW_CONVLSTM_WIDE * Config.FWBW_CONVLSTM_HIGH))
# if not os.path.isfile(Config.RESULTS_PATH + 'ground_true_{}.npy'.format(data_name)):
# np.save(Config.RESULTS_PATH + 'ground_true_{}.npy'.format(data_name),
# test_data2d)
# if not os.path.isfile(Config.RESULTS_PATH + 'ground_true_scaled_{}_{}.npy'.format(data_name, Config.SCALER)):
# np.save(Config.RESULTS_PATH + 'ground_true_scaled_{}_{}.npy'.format(data_name, Config.SCALER),
# test_data_normalized2d)
if not os.path.exists(Config.RESULTS_PATH + '{}-{}-{}-{}/'.format(data_name,
alg_name, tag, Config.SCALER)):
os.makedirs(Config.RESULTS_PATH + '{}-{}-{}-{}/'.format(data_name, alg_name, tag, Config.SCALER))
for i in range(Config.FWBW_CONVLSTM_TESTING_TIME):
print('|--- Run time {}'.format(i))
init_data = np.reshape(init_data2d, newshape=(init_data2d.shape[0],
Config.FWBW_CONVLSTM_WIDE,
Config.FWBW_CONVLSTM_HIGH))
test_data_normalized = np.reshape(test_data_normalized2d, newshape=(test_data_normalized2d.shape[0],
Config.FWBW_CONVLSTM_WIDE,
Config.FWBW_CONVLSTM_HIGH))
pred_tm, measured_matrix, ims_tm = predict_fwbw_convlstm(initial_data=init_data,
test_data=test_data_normalized,
model=net.model)
pred_tm2d = np.reshape(np.copy(pred_tm), newshape=(pred_tm.shape[0], pred_tm.shape[1] * pred_tm.shape[2]))
measured_matrix2d = np.reshape(np.copy(measured_matrix),
newshape=(measured_matrix.shape[0],
measured_matrix.shape[1] * measured_matrix.shape[2]))
# np.save(Config.RESULTS_PATH + '{}-{}-{}-{}/pred_scaled-{}.npy'.format(data_name, alg_name, tag,
# Config.SCALER, i),
# pred_tm2d)
pred_tm_invert2d = scalers.inverse_transform(pred_tm2d)
if np.any(np.isinf(pred_tm_invert2d)):
raise ValueError('Value is infinity!')
elif np.any(np.isnan(pred_tm_invert2d)):
raise ValueError('Value is NaN!')
err.append(error_ratio(y_true=test_data2d, y_pred=pred_tm_invert2d, measured_matrix=measured_matrix2d))
r2_score.append(calculate_r2_score(y_true=test_data2d, y_pred=pred_tm_invert2d))
rmse.append(calculate_rmse(y_true=test_data2d / 1000000, y_pred=pred_tm_invert2d / 1000000))
if Config.FWBW_IMS:
# Calculate error for multistep-ahead-prediction
ims_tm2d = np.reshape(np.copy(ims_tm), newshape=(ims_tm.shape[0], ims_tm.shape[1] * ims_tm.shape[2]))
ims_tm_invert2d = scalers.inverse_transform(ims_tm2d)
ims_ytrue2d = ims_tm_test_data(test_data=test_data2d)
err_ims.append(error_ratio(y_pred=ims_tm_invert2d,
y_true=ims_ytrue2d,
measured_matrix=measured_matrix_ims2d))
r2_score_ims.append(calculate_r2_score(y_true=ims_ytrue2d, y_pred=ims_tm_invert2d))
rmse_ims.append(calculate_rmse(y_true=ims_ytrue2d / 1000000, y_pred=ims_tm_invert2d / 1000000))
else:
err_ims.append(0)
r2_score_ims.append(0)
rmse_ims.append(0)
print('Result: err\trmse\tr2 \t\t err_ims\trmse_ims\tr2_ims')
print(' {}\t{}\t{} \t\t {}\t{}\t{}'.format(err[i], rmse[i], r2_score[i],
err_ims[i], rmse_ims[i],
r2_score_ims[i]))
# np.save(Config.RESULTS_PATH + '{}-{}-{}-{}/pred-{}.npy'.format(data_name, alg_name, tag,
# Config.SCALER, i),
# pred_tm_invert2d)
# np.save(Config.RESULTS_PATH + '{}-{}-{}-{}/measure-{}.npy'.format(data_name, alg_name, tag,
# Config.SCALER, i),
# measured_matrix2d)
results_summary['No.'] = range(Config.FWBW_CONVLSTM_TESTING_TIME)
results_summary['err'] = err
results_summary['r2'] = r2_score
results_summary['rmse'] = rmse
results_summary['err_ims'] = err_ims
results_summary['r2_ims'] = r2_score_ims
results_summary['rmse_ims'] = rmse_ims
results_summary.to_csv(Config.RESULTS_PATH +
'{}-{}-{}-{}/results.csv'.format(data_name, alg_name, tag, Config.SCALER),
index=False)
print('Test: {}-{}-{}-{}'.format(data_name, alg_name, tag, Config.SCALER))
print('avg_err: {} - avg_rmse: {} - avg_r2: {}'.format(np.mean(np.array(err)),
np.mean(np.array(rmse)),
np.mean(np.array(r2_score))))
return