def eval_var(traffic_reading_df, n_lags=3):
n_forwards = [1, 2]
y_predicts, y_test = var_predict(traffic_reading_df,
n_forwards=n_forwards,
n_lags=n_lags,
test_ratio=0.2)
#print("y_predicts type:" + types(y_predicts))
print("type===========")
print(type(y_predicts[1].as_matrix()))
#df_list = pd.DataFrame(y_predicts)
#df_list.to_csv("./hello.csv", sep=',', index=False)
logger.info('VAR (lag=%d)' % n_lags)
logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE')
for i, horizon in enumerate(n_forwards):
rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mape = masked_mape_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mae = masked_mae_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def train(self):
training_history = self.model.fit(
x=self._data['x_train'],
y=self._data['y_train'],
batch_size=self._batch_size,
epochs=self._epochs,
callbacks=self.callbacks_list,
validation_data=(self._data['x_val'], self._data['y_val']),
shuffle=True,
verbose=2)
if training_history is not None:
self._plot_training_history(training_history)
self._save_model_history(training_history)
config = dict(self._kwargs)
config_filename = 'config_lstm.yaml'
config['train']['log_dir'] = self._log_dir
with open(os.path.join(self._log_dir, config_filename), 'w') as f:
yaml.dump(config, f, default_flow_style=False)
# evaluate
scaler = self._data['scaler']
x_eval = self._data['x_eval']
y_truth = self._data['y_eval']
y_pred = self.model.predict(x_eval)
y_pred = scaler.inverse_transform(y_pred)
y_truth = scaler.inverse_transform(y_truth)
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format(
1, mse, mape, rmse))
def evaluate(model,
dataset,
dataset_type,
edge_index,
edge_attr,
device,
output_dim,
logger,
detail=True,
cfg=None,
format_result=False):
if detail:
logger.info('Evaluation_{}_Begin:'.format(dataset_type))
scaler = dataset['scaler']
y_preds = run_model(
model,
data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(),
edge_index=edge_index,
edge_attr=edge_attr,
device=device,
output_dim=output_dim)
y_preds = np.concatenate(y_preds, axis=0) # concat in batch_size dim.
mae_list = []
mape_list = []
rmse_list = []
mae_sum = 0
mape_sum = 0
rmse_sum = 0
# horizon = dataset['y_{}'.format(dataset_type)].shape[1]
horizon = cfg['model']['horizon']
for horizon_i in range(horizon):
y_truth = scaler.inverse_transform(
dataset['y_{}'.format(dataset_type)][:, horizon_i, :, :output_dim])
y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0],
horizon_i, :, :output_dim])
mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn')
mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
mae_sum += mae
mape_sum += mape
rmse_sum += rmse
mae_list.append(mae)
mape_list.append(mape)
rmse_list.append(rmse)
msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}"
if detail:
logger.info(msg.format(horizon_i + 1, mae, mape, rmse))
if detail:
logger.info('Evaluation_{}_End:'.format(dataset_type))
if format_result:
for i in range(len(mape_list)):
print('{:.2f}'.format(mae_list[i]))
print('{:.2f}%'.format(mape_list[i] * 100))
print('{:.2f}'.format(rmse_list[i]))
print()
else:
return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
def test_masked_rmse_np2(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([[1, 0], [3, 3]], dtype=np.float32)
rmse = metrics.masked_rmse_np(preds=preds, labels=labels, null_val=0)
self.assertAlmostEqual(np.sqrt(1 / 3.), rmse, delta=1e-5)
def test_masked_rmse_np_missing(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([[1, 0], [3, 4]], dtype=np.float32)
mape = metrics.masked_rmse_np(preds=preds, labels=labels, null_val=0)
self.assertEqual(0., mape)
def test_masked_rmse_np_nan(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([[1, np.nan], [3, 4]], dtype=np.float32)
rmse = metrics.masked_rmse_np(preds=preds, labels=labels)
self.assertEqual(0., rmse)
def eval_historical_average(traffic_reading_df, period):
y_predict, y_test = historical_average_predict(traffic_reading_df, period=period, test_ratio=0.2)
rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
logger.info('Historical Average')
logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
for horizon in [1, 3, 6, 12]:
line = 'HA\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def eval_static(traffic_reading_df):
logger.info('Static')
horizons = [1, 3, 6, 12]
logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
for horizon in horizons:
y_predict, y_test = static_predict(traffic_reading_df, n_forward=horizon, test_ratio=0.2)
rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
line = 'Static\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def eval_var(traffic_reading_df, n_lags=3):
n_forwards = [1, 3, 6, 12]
y_predicts, y_test = var_predict(traffic_reading_df, n_forwards=n_forwards, n_lags=n_lags,
test_ratio=0.2)
logger.info('VAR (lag=%d)' % n_lags)
logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE')
for i, horizon in enumerate(n_forwards):
rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
mape = masked_mape_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
mae = masked_mae_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def evaluate(self):
scaler = self._data['scaler']
y_pred_1, y_pred_2 = self.model.predict(self._data['x_eval'])
y_pred_1 = scaler.inverse_transform(y_pred_1)
y_truth_1 = scaler.inverse_transform(self._data['y_eval_1'])
y_truth_2 = scaler.inverse_transform(self._data['y_eval_2'])
mse = metrics.masked_mse_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
mae = metrics.masked_mae_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
mape = metrics.masked_mape_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
self._logger.info(
" Forward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(mse, mae, rmse, mape))
mse_2 = metrics.masked_mse_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
mae_2 = metrics.masked_mae_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
mape_2 = metrics.masked_mape_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
rmse_2 = metrics.masked_rmse_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
self._logger.info(
"Backward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(mse_2, mae_2, rmse_2, mape_2))
def evaluate(self):
scaler = self._data['scaler']
y_pred = self.model.predict(self._data['x_eval'])
y_pred = scaler.inverse_transform(y_pred)
y_truth = scaler.inverse_transform(self._data['y_eval'])
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(1, mse, mae, rmse, mape))
def evaluate(self, sess):
global_step = sess.run(tf.train.get_or_create_global_step())
test_results = self.run_epoch_generator(
sess,
self.model,
self._data['eval_loader'].get_iterator(),
return_output=True,
training=False)
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
test_loss, y_preds = test_results['loss'], test_results['outputs']
utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss],
global_step=global_step)
y_preds = np.concatenate(y_preds, axis=0)
scaler = self._data['scaler']
predictions = []
y_truths = []
for horizon_i in range(self._data['y_eval'].shape[1]):
y_truth = scaler.inverse_transform(
self._data['y_eval'][:, horizon_i, :, 0])
y_truths.append(y_truth)
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred,
labels=y_truth,
null_val=0)
mae = metrics.masked_mae_np(preds=y_pred,
labels=y_truth,
null_val=0)
mape = metrics.masked_mape_np(preds=y_pred,
labels=y_truth,
null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred,
labels=y_truth,
null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(horizon_i + 1, mse, mae, rmse, mape))
utils.add_simple_summary(self._writer, [
'%s_%d' % (item, horizon_i + 1)
for item in ['metric/rmse', 'metric/mae', 'metric/mse']
], [rmse, mae, mse],
global_step=global_step)
outputs = {'predictions': predictions, 'groundtruth': y_truths}
return outputs
def _calculate_metrics(self, prediction_results, metrics_summary, scaler, runId, data_norm, n_metrics=4):
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
y_preds = prediction_results['y_preds']
y_preds = np.concatenate(y_preds, axis=0)
y_truths = prediction_results['y_truths']
y_truths = np.concatenate(y_truths, axis=0)
predictions = []
for horizon_i in range(self._horizon):
y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :])
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format(
horizon_i + 1, mse, mae, rmse, mape
)
)
metrics_summary[runId, horizon_i * n_metrics + 0] = mse
metrics_summary[runId, horizon_i * n_metrics + 1] = mae
metrics_summary[runId, horizon_i * n_metrics + 2] = rmse
metrics_summary[runId, horizon_i * n_metrics + 3] = mape
tm_pred = scaler.inverse_transform(prediction_results['tm_pred'])
g_truth = scaler.inverse_transform(data_norm[self._seq_len:-self._horizon])
m_indicator = prediction_results['m_indicator']
er = metrics.error_ratio(y_pred=tm_pred,
y_true=g_truth,
measured_matrix=m_indicator)
metrics_summary[runId, -1] = er
self._logger.info('ER: {}'.format(er))
self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(runId))
return metrics_summary
def _test(self):
scaler = self._data['scaler']
results_summary = pd.DataFrame(index=range(self._run_times))
results_summary['No.'] = range(self._run_times)
n_metrics = 4
# Metrics: MSE, MAE, RMSE, MAPE, ER
metrics_summary = np.zeros(shape=(self._run_times,
self._horizon * n_metrics + 1))
for i in range(self._run_times):
print('|--- Running time: {}/{}'.format(i, self._run_times))
outputs = self._run_tm_prediction()
tm_pred, m_indicator, y_preds = outputs['tm_pred'], outputs[
'm_indicator'], outputs['y_preds']
y_preds = np.concatenate(y_preds, axis=0)
predictions = []
y_truths = outputs['y_truths']
y_truths = np.concatenate(y_truths, axis=0)
for horizon_i in range(self._horizon):
y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :])
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred,
labels=y_truth,
null_val=0)
mae = metrics.masked_mae_np(preds=y_pred,
labels=y_truth,
null_val=0)
mape = metrics.masked_mape_np(preds=y_pred,
labels=y_truth,
null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred,
labels=y_truth,
null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(horizon_i + 1, mse, mae, rmse, mape))
metrics_summary[i, horizon_i * n_metrics + 0] = mse
metrics_summary[i, horizon_i * n_metrics + 1] = mae
metrics_summary[i, horizon_i * n_metrics + 2] = rmse
metrics_summary[i, horizon_i * n_metrics + 3] = mape
tm_pred = scaler.inverse_transform(tm_pred)
g_truth = scaler.inverse_transform(
self._data['test_data_norm'][self._seq_len:-self._horizon])
er = metrics.error_ratio(y_pred=tm_pred,
y_true=g_truth,
measured_matrix=m_indicator)
metrics_summary[i, -1] = er
self._save_results(g_truth=g_truth,
pred_tm=tm_pred,
m_indicator=m_indicator,
tag=str(i))
print('ER: {}'.format(er))
for horizon_i in range(self._horizon):
results_summary['mse_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0]
results_summary['mae_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1]
results_summary['rmse_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2]
results_summary['mape_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3]
results_summary['er'] = metrics_summary[:, -1]
results_summary.to_csv(self._log_dir + 'results_summary.csv',
index=False)
def _test(self, sess, **kwargs):
global_step = sess.run(tf.train.get_or_create_global_step())
results_summary = pd.DataFrame(index=range(self._run_times))
results_summary['No.'] = range(self._run_times)
n_metrics = 4
# Metrics: MSE, MAE, RMSE, MAPE, ER
metrics_summary = np.zeros(shape=(self._run_times, self._horizon * n_metrics + 1))
for i in range(self._run_times):
self._logger.info('|--- Run time: {}'.format(i))
# y_test = self._prepare_test_set()
test_results = self._run_tm_prediction(sess, model=self._test_model)
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
test_loss, y_preds = test_results['loss'], test_results['y_preds']
utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step)
y_preds = test_results['y_preds']
y_preds = np.concatenate(y_preds, axis=0)
y_truths = test_results['y_truths']
y_truths = np.concatenate(y_truths, axis=0)
scaler = self._data['scaler']
predictions = []
for horizon_i in range(self._horizon):
y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :, 0])
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format(
horizon_i + 1, mse, mae, rmse, mape
)
)
metrics_summary[i, horizon_i * n_metrics + 0] = mse
metrics_summary[i, horizon_i * n_metrics + 1] = mae
metrics_summary[i, horizon_i * n_metrics + 2] = rmse
metrics_summary[i, horizon_i * n_metrics + 3] = mape
tm_pred = scaler.inverse_transform(test_results['tm_pred'])
g_truth = scaler.inverse_transform(self._data['test_data_norm'][self._seq_len:-self._horizon])
m_indicator = test_results['m_indicator']
er = error_ratio(y_pred=tm_pred,
y_true=g_truth,
measured_matrix=m_indicator)
metrics_summary[i, -1] = er
self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(i))
print('ER: {}'.format(er))
for horizon_i in range(self._horizon):
results_summary['mse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0]
results_summary['mae_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1]
results_summary['rmse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2]
results_summary['mape_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3]
results_summary['er'] = metrics_summary[:, -1]
results_summary.to_csv(self._log_dir + 'results_summary.csv', index=False)
return
def evaluate(model,
dataset,
dataset_type,
edge_index,
edge_attr,
device,
output_dim,
logger,
detail=True,
cfg=None,
format_result=False):
if detail:
logger.info('Evaluation_{}_Begin:'.format(dataset_type))
scaler = dataset['scaler']
y_preds = run_model(
model,
data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(),
edge_index=edge_index,
edge_attr=edge_attr,
device=device,
output_dim=output_dim)
y_preds = np.concatenate(y_preds, axis=0) # concat in batch_size dim.
mae_sum = 0
mape_sum = 0
rmse_sum = 0
mae_list = []
mape_list = []
rmse_list = []
mae_peak_list = []
mape_peak_list = []
rmse_peak_list = []
# horizon = dataset['y_{}'.format(dataset_type)].shape[1]
horizon = cfg['model']['horizon']
ytime = dataset['ytime_{}'.format(dataset_type)][:, 0]
ytime_pd = pd.to_datetime(ytime.flatten())
morning_peak = np.logical_and(
(ytime_pd.hour * 60 + ytime_pd.minute) >= 7 * 60 + 30,
(ytime_pd.hour * 60 + ytime_pd.minute) <= 8 * 60 + 30)
evening_peak = np.logical_and(
(ytime_pd.hour * 60 + ytime_pd.minute) >= 17 * 60 + 30,
(ytime_pd.hour * 60 + ytime_pd.minute) <= 18 * 60 + 30)
peak = np.logical_or(morning_peak, evening_peak)
for horizon_i in range(horizon):
y_truth = scaler.inverse_transform(
dataset['y_{}'.format(dataset_type)][:, horizon_i,
shtop72_index, :output_dim])
y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0], horizon_i,
shtop72_index, :output_dim])
mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn')
mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
mae_sum += mae
mape_sum += mape
rmse_sum += rmse
mae_list.append(mae)
mape_list.append(mape)
rmse_list.append(rmse)
# Peak
y_truth_peak = scaler.inverse_transform(
dataset['y_{}'.format(dataset_type)][peak,
horizon_i, :, :output_dim])
y_pred_peak = scaler.inverse_transform(
y_preds[:y_truth.shape[0], horizon_i, :, :output_dim])
y_pred_peak = y_pred_peak[peak]
mae_peak = metrics.masked_mae_np(y_pred_peak,
y_truth_peak,
null_val=0,
mode='dcrnn')
mape_peak = metrics.masked_mape_np(y_pred_peak,
y_truth_peak,
null_val=0)
rmse_peak = metrics.masked_rmse_np(y_pred_peak,
y_truth_peak,
null_val=0)
mae_peak_list.append(mae_peak)
mape_peak_list.append(mape_peak)
rmse_peak_list.append(rmse_peak)
msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}"
if detail:
logger.info(msg.format(horizon_i + 1, mae, mape, rmse))
if detail:
logger.info('Evaluation_{}_End:'.format(dataset_type))
if format_result:
print('-' * 40 + ' 1/4 ' + '-' * 40)
for i in range(len(mape_list)):
print('{:.2f}'.format(mae_list[i]))
print('{:.2f}%'.format(mape_list[i] * 100))
print('{:.2f}'.format(rmse_list[i]))
print()
print('-' * 40 + ' Peak ' + '-' * 40)
for i in range(len(mape_list)):
print('{:.2f}'.format(mae_peak_list[i]))
print('{:.2f}%'.format(mape_peak_list[i] * 100))
print('{:.2f}'.format(rmse_peak_list[i]))
print()
else:
return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
def _train(self,
sess,
base_lr,
epoch,
steps,
patience=50,
epochs=100,
min_learning_rate=2e-6,
lr_decay_ratio=0.1,
save_model=1,
test_every_n_epochs=10,
**train_kwargs):
history = []
min_val_loss = float('inf')
wait = 0
max_to_keep = train_kwargs.get('max_to_keep', 100)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep)
model_filename = train_kwargs.get('model_filename')
output_file = train_kwargs.get('preds_file')
gt_file = train_kwargs.get('groundtruth_file')
if model_filename is not None:
saver.restore(sess, model_filename)
self._epoch = epoch + 1
else:
sess.run(tf.global_variables_initializer())
self._logger.info('Start training ...')
while self._epoch <= epochs:
# Learning rate schedule.
new_lr = max(
min_learning_rate,
base_lr *
(lr_decay_ratio**np.sum(self._epoch >= np.array(steps))))
self.set_lr(sess=sess, lr=new_lr)
start_time = time.time()
train_results = self.run_epoch_generator(
sess,
self._train_model,
self._data['train_loader'].get_iterator(),
training=True,
writer=self._writer)
train_loss, train_mae, train_reg = train_results[
'loss'], train_results['mae'], train_results['reg']
#print ('reg loss is:', train_reg)
if train_loss > 1e5:
self._logger.warning('Gradient explosion detected. Ending...')
break
global_step = sess.run(tf.train.get_or_create_global_step())
# Compute validation error.
val_results = self.run_epoch_generator(
sess,
self._test_model,
self._data['val_loader'].get_iterator(),
training=False)
val_loss, val_mae = np.asscalar(val_results['loss']), np.asscalar(
val_results['mae'])
utils.add_simple_summary(self._writer, [
'loss/train_loss', 'metric/train_mae', 'loss/val_loss',
'metric/val_mae'
], [train_loss, train_mae, val_loss, val_mae],
global_step=global_step)
end_time = time.time()
message = 'Epoch [{}/{}] ({}) train_mae: {:.4f}, val_mae: {:.4f} lr:{:.6f} {:.1f}s'.format(
self._epoch, epochs, global_step, train_mae, val_mae, new_lr,
(end_time - start_time))
self._logger.info(message)
stt = time.time()
outputs = self.evaluate(sess)
#print (outputs['groundtruth'].shape)
test_gdt = outputs['groundtruth'][:, :, :, 0]
test_y = outputs['observed'][:, :, :, 0]
best_pred = outputs['predictions'][:test_gdt.shape[0], :, :, 0]
test_gdt = test_gdt.reshape(
test_gdt.shape[0], -1, order='F'
) # record the corresponding steps for each node first, then node by node
test_y = test_y.reshape(test_gdt.shape[0], -1, order='F')
best_pred = best_pred.reshape(test_gdt.shape[0], -1, order='F')
scaler = self._data['scaler']
print('Test running time: %fs' % (time.time() - stt))
# --------
# best_pred = scaler.inverse_transform(best_pred)
# # np.save('best_pred.npy', best_pred)
# mape = metrics.masked_mape_np(best_pred, test_y, test_gdt, null_val=0)
# rmse = metrics.masked_rmse_np(best_pred, test_y, test_gdt, null_val=0)
# self._logger.info(
# 'Overall Test MAPE %.4f, RMSE %.4f' % (mape, rmse))
# --------
if val_loss <= min_val_loss:
wait = 0
if save_model > 0:
model_filename = self.save(sess, val_loss)
self._logger.info(
'Val loss decrease from %.4f to %.4f, saving to %s' %
(min_val_loss, val_loss, model_filename))
min_val_loss = val_loss
test_y = scaler.inverse_transform(test_y)
best_pred = scaler.inverse_transform(best_pred)
# np.save('best_pred.npy', best_pred)
mape = metrics.masked_mape_np(best_pred,
test_y,
test_gdt,
null_val=0)
rmse = metrics.masked_rmse_np(best_pred,
test_y,
test_gdt,
null_val=0)
self._logger.info('Overall Test MAPE %.4f, RMSE %.4f' %
(mape, rmse))
print(best_pred.shape)
#np.savetxt(output_file, best_pred, delimiter = ',')
#np.savetxt(gt_file, test_gdt, delimiter = ',')
else:
wait += 1
if wait > patience:
self._logger.warning('Early stopping at epoch: %d' %
self._epoch)
break
# test_y = scaler.inverse_transform(test_y)
# best_pred = scaler.inverse_transform(best_pred)
# mape = metrics.masked_mape_np(best_pred, test_y, test_gdt, null_val=0)
# rmse = metrics.masked_rmse_np(best_pred, test_y, test_gdt, null_val=0)
# self._logger.info(
# 'Overall Test MAPE %.4f, RMSE %.4f' % (mape, rmse))
history.append(val_mae)
# Increases epoch.
self._epoch += 1
sys.stdout.flush()
return np.min(history)
def evaluate(self, sess, **kwargs):
global_step = sess.run(tf.train.get_or_create_global_step())
test_results = self.run_epoch_generator(
sess,
self._test_model,
self._data['test_loader'].get_iterator(),
return_output=True,
training=False,
return_ground_truth=True,
task='test')
test_loss = test_results['loss']
# utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss],
# global_step=global_step)
utils.add_summary_dict(self._writer, {
'loss/test_loss': test_loss,
},
global_step=global_step)
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
y_preds = np.concatenate(test_results['outputs'], axis=0)
scaler = self._data['scaler']
y_truths = np.concatenate(test_results['ground_truths'], axis=0)
assert y_preds.shape[:3] == y_truths.shape[:3], \
'NOT {} == {}'.format(y_preds.shape[:3], y_truths.shape[:3])
y_preds_original = []
y_truths_original = []
min_output_value = self._data_kwargs.get('min_output_value')
max_output_value = self._data_kwargs.get('max_output_value')
clip = (min_output_value is not None) or (max_output_value is not None)
if clip:
self._kwargs.logger.info('The output values are clipped to range: [{}, {}]'.\
format(min_output_value, max_output_value))
for horizon_i in range(y_truths.shape[1]):
y_pred_original = scaler.inverse_transform(y_preds[:, horizon_i, :,
0])
if clip:
y_pred_original = \
np.clip(y_pred_original, min_output_value, max_output_value,
out=y_pred_original)
y_preds_original.append(y_pred_original)
self._kwargs.logger.info(
stat_str(y_pred_original, horizon_i, 'pred'))
y_truth_original = scaler.inverse_transform(y_truths[:,
horizon_i, :,
0])
y_truths_original.append(y_truth_original)
if not np.all(np.isnan(y_truth_original)):
self._kwargs.logger.info(
stat_str(y_truth_original, horizon_i, 'true'))
for null_val in [0, np.nan]:
desc = r'0 excl.' if null_val == 0 else 'any '
rmse = metrics.masked_rmse_np(y_pred_original,
y_truth_original,
null_val=null_val)
r2 = metrics.masked_r2_np(y_pred_original,
y_truth_original,
null_val=null_val)
mae = metrics.masked_mae_np(y_pred_original,
y_truth_original,
null_val=null_val)
mape = metrics.masked_mape_np(y_pred_original, y_truth_original, null_val=null_val) \
if null_val == 0 else np.nan
self._kwargs.logger.info(
"T+{:02d}|{}|RMSE: {:8.5f}|R2: {:8.4f}|MAE: {:5.2f}|MAPE: {:5.3f}|".\
format(horizon_i + 1, desc, rmse, r2, mae, mape,)
)
# utils.add_simple_summary(self._writer,
# ['%s_%d' % (item, horizon_i + 1) for item in
# ['metric/rmse', 'metric/mape', 'metric/mae']],
# [rmse, mape, mae],
# global_step=global_step)
utils.add_summary_dict(
self._writer, {
'metric/rmse_horizon{}'.format(horizon_i + 1): rmse,
'metric/mape_horizon{}'.format(horizon_i + 1): mape,
'metric/mae_horizon{}'.format(horizon_i + 1): mae,
'metric/r2_horizon{}'.format(horizon_i + 1): r2,
},
global_step=global_step)
outputs = {
'predictions': y_preds_original,
'groundtruth': y_truths_original
}
return outputs
def main(config):
logger = config.get_logger('test')
graph_pkl_filename = 'data/sensor_graph/adj_mx_unix.pkl'
_, _, adj_mat = utils.load_graph_data(graph_pkl_filename)
data = utils.load_dataset(dataset_dir='data/METR-LA',
batch_size=config["arch"]["args"]["batch_size"],
test_batch_size=config["arch"]["args"]["batch_size"])
test_data_loader = data['test_loader']
scaler = data['scaler']
num_test_iteration= math.ceil(data['x_test'].shape[0] / config["arch"]["args"]["batch_size"])
# build model architecture
adj_arg = {"adj_mat": adj_mat}
model = config.initialize('arch', module_arch, **adj_arg)
logger.info(model)
logger.info('Loading checkpoint: {} ...'.format(config.resume))
checkpoint = torch.load(config.resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
# prepare model for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
total_loss = 0.0
y_preds = torch.FloatTensor([])
y_truths = data['y_test'] # (6850, 12, 207, 2)
y_truths = scaler.inverse_transform(y_truths)
predictions = []
groundtruth = list()
with torch.no_grad():
for i, (x, y) in tqdm(enumerate(test_data_loader.get_iterator()), total=num_test_iteration):
x = torch.FloatTensor(x).cuda()
y = torch.FloatTensor(y).cuda()
outputs = model(x, y, 0) # (seq_length+1, batch_size, num_nodes*output_dim) (13, 50, 207*1)
y_preds = torch.cat([y_preds, outputs], dim=1)
y_preds = torch.transpose(y_preds, 0, 1)
y_preds = y_preds.detach().numpy() # cast to numpy array
print("--------test results--------")
for horizon_i in range(y_truths.shape[1]):
y_truth = np.squeeze(y_truths[:, horizon_i, :, 0]) # (6850, 207)
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :]) # (6850, 207)
predictions.append(y_pred)
groundtruth.append(y_truth)
mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0)
mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
print(
"Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format(
horizon_i + 1, mae, mape, rmse
)
)
log = {"Horizon": horizon_i+1, "MAE": mae, "MAPE": mape, "RMSE": rmse}
logger.info(log)
outputs = {
'predictions': predictions,
'groundtruth': groundtruth
}
# serialize test data
np.savez_compressed('saved/results/dcrnn_predictions.npz', **outputs)
print('Predictions saved as {}.'.format('saved/results/dcrnn_predictions.npz'))
