def train(self):
training_history = self.model.fit(
x=self._data['x_train'],
y=self._data['y_train'],
batch_size=self._batch_size,
epochs=self._epochs,
callbacks=self.callbacks_list,
validation_data=(self._data['x_val'], self._data['y_val']),
shuffle=True,
verbose=2)
if training_history is not None:
self._plot_training_history(training_history)
self._save_model_history(training_history)
config = dict(self._kwargs)
config_filename = 'config_lstm.yaml'
config['train']['log_dir'] = self._log_dir
with open(os.path.join(self._log_dir, config_filename), 'w') as f:
yaml.dump(config, f, default_flow_style=False)
# evaluate
scaler = self._data['scaler']
x_eval = self._data['x_eval']
y_truth = self._data['y_eval']
y_pred = self.model.predict(x_eval)
y_pred = scaler.inverse_transform(y_pred)
y_truth = scaler.inverse_transform(y_truth)
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format(
1, mse, mape, rmse))
def eval_var(traffic_reading_df, n_lags=3):
n_forwards = [1, 2]
y_predicts, y_test = var_predict(traffic_reading_df,
n_forwards=n_forwards,
n_lags=n_lags,
test_ratio=0.2)
#print("y_predicts type:" + types(y_predicts))
print("type===========")
print(type(y_predicts[1].as_matrix()))
#df_list = pd.DataFrame(y_predicts)
#df_list.to_csv("./hello.csv", sep=',', index=False)
logger.info('VAR (lag=%d)' % n_lags)
logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE')
for i, horizon in enumerate(n_forwards):
rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mape = masked_mape_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mae = masked_mae_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def evaluate(model,
dataset,
dataset_type,
edge_index,
edge_attr,
device,
output_dim,
logger,
detail=True,
cfg=None,
format_result=False):
if detail:
logger.info('Evaluation_{}_Begin:'.format(dataset_type))
scaler = dataset['scaler']
y_preds = run_model(
model,
data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(),
edge_index=edge_index,
edge_attr=edge_attr,
device=device,
output_dim=output_dim)
y_preds = np.concatenate(y_preds, axis=0) # concat in batch_size dim.
mae_list = []
mape_list = []
rmse_list = []
mae_sum = 0
mape_sum = 0
rmse_sum = 0
# horizon = dataset['y_{}'.format(dataset_type)].shape[1]
horizon = cfg['model']['horizon']
for horizon_i in range(horizon):
y_truth = scaler.inverse_transform(
dataset['y_{}'.format(dataset_type)][:, horizon_i, :, :output_dim])
y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0],
horizon_i, :, :output_dim])
mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn')
mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
mae_sum += mae
mape_sum += mape
rmse_sum += rmse
mae_list.append(mae)
mape_list.append(mape)
rmse_list.append(rmse)
msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}"
if detail:
logger.info(msg.format(horizon_i + 1, mae, mape, rmse))
if detail:
logger.info('Evaluation_{}_End:'.format(dataset_type))
if format_result:
for i in range(len(mape_list)):
print('{:.2f}'.format(mae_list[i]))
print('{:.2f}%'.format(mape_list[i] * 100))
print('{:.2f}'.format(rmse_list[i]))
print()
else:
return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
def test_masked_mape_np(self):
preds = np.array([
[1, 2, 2],
[3, 4, 5],
], dtype=np.float32)
labels = np.array([[1, 2, 2], [3, 4, 4]], dtype=np.float32)
mape = metrics.masked_mape_np(preds=preds, labels=labels)
self.assertAlmostEqual(1 / 24.0, mape, delta=1e-5)
def test_masked_mape_np_nan(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([[np.nan, np.nan], [np.nan, 3]], dtype=np.float32)
mape = metrics.masked_mape_np(preds=preds, labels=labels)
self.assertAlmostEqual(1 / 3., mape, delta=1e-5)
def test_masked_mape_np_all_zero(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([[0, 0], [0, 0]], dtype=np.float32)
mape = metrics.masked_mape_np(preds=preds, labels=labels, null_val=0)
self.assertEqual(0., mape)
def test_masked_mape_np2(self):
preds = np.array([
[1, 2, 2],
[3, 4, 5],
], dtype=np.float32)
labels = np.array([[1, 2, 2], [3, 4, 4]], dtype=np.float32)
mape = metrics.masked_mape_np(preds=preds, labels=labels, null_val=4)
self.assertEqual(0., mape)
def test_masked_mape_np_all_nan(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([[np.nan, np.nan], [np.nan, np.nan]],
dtype=np.float32)
mape = metrics.masked_mape_np(preds=preds, labels=labels)
self.assertEqual(0., mape)
def evaluate_on_test_mstgcn(net, test_loader, test_target_tensor, sw, epoch,
_mean, _std):
'''
for rnn, compute MAE, RMSE, MAPE scores of the prediction for every time step on testing set.
:param net: model
:param test_loader: torch.utils.data.utils.DataLoader
:param test_target_tensor: torch.tensor (B, N_nodes, T_output, out_feature)=(B, N_nodes, T_output, 1)
:param sw:
:param epoch: int, current epoch
:param _mean: (1, 1, 3(features), 1)
:param _std: (1, 1, 3(features), 1)
'''
net.train(False) # ensure dropout layers are in test mode
with torch.no_grad():
test_loader_length = len(test_loader)
test_target_tensor = test_target_tensor.cpu().numpy()
prediction = [] # 存储所有batch的output
for batch_index, batch_data in enumerate(test_loader):
encoder_inputs, labels = batch_data
outputs = net(encoder_inputs)
prediction.append(outputs.detach().cpu().numpy())
if batch_index % 100 == 0:
print('predicting testing set batch %s / %s' %
(batch_index + 1, test_loader_length))
prediction = np.concatenate(prediction, 0) # (batch, T', 1)
prediction_length = prediction.shape[2]
for i in range(prediction_length):
assert test_target_tensor.shape[0] == prediction.shape[0]
print('current epoch: %s, predict %s points' % (epoch, i))
mae = mean_absolute_error(test_target_tensor[:, :, i],
prediction[:, :, i])
rmse = mean_squared_error(test_target_tensor[:, :, i],
prediction[:, :, i])**0.5
mape = masked_mape_np(test_target_tensor[:, :, i],
prediction[:, :, i], 0)
print('MAE: %.2f' % (mae))
print('RMSE: %.2f' % (rmse))
print('MAPE: %.2f' % (mape))
print()
if sw:
sw.add_scalar('MAE_%s_points' % (i), mae, epoch)
sw.add_scalar('RMSE_%s_points' % (i), rmse, epoch)
sw.add_scalar('MAPE_%s_points' % (i), mape, epoch)
def eval_historical_average(traffic_reading_df, period):
y_predict, y_test = historical_average_predict(traffic_reading_df, period=period, test_ratio=0.2)
rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
logger.info('Historical Average')
logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
for horizon in [1, 3, 6, 12]:
line = 'HA\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def eval_static(traffic_reading_df):
logger.info('Static')
horizons = [1, 3, 6, 12]
logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
for horizon in horizons:
y_predict, y_test = static_predict(traffic_reading_df, n_forward=horizon, test_ratio=0.2)
rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
line = 'Static\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def eval_var(traffic_reading_df, n_lags=3):
n_forwards = [1, 3, 6, 12]
y_predicts, y_test = var_predict(traffic_reading_df, n_forwards=n_forwards, n_lags=n_lags,
test_ratio=0.2)
logger.info('VAR (lag=%d)' % n_lags)
logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE')
for i, horizon in enumerate(n_forwards):
rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
mape = masked_mape_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
mae = masked_mae_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def evaluate(self):
scaler = self._data['scaler']
y_pred_1, y_pred_2 = self.model.predict(self._data['x_eval'])
y_pred_1 = scaler.inverse_transform(y_pred_1)
y_truth_1 = scaler.inverse_transform(self._data['y_eval_1'])
y_truth_2 = scaler.inverse_transform(self._data['y_eval_2'])
mse = metrics.masked_mse_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
mae = metrics.masked_mae_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
mape = metrics.masked_mape_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred_1,
labels=y_truth_1,
null_val=0)
self._logger.info(
" Forward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(mse, mae, rmse, mape))
mse_2 = metrics.masked_mse_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
mae_2 = metrics.masked_mae_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
mape_2 = metrics.masked_mape_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
rmse_2 = metrics.masked_rmse_np(preds=y_pred_2,
labels=y_truth_2,
null_val=0)
self._logger.info(
"Backward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(mse_2, mae_2, rmse_2, mape_2))
def evaluate(self):
scaler = self._data['scaler']
y_pred = self.model.predict(self._data['x_eval'])
y_pred = scaler.inverse_transform(y_pred)
y_truth = scaler.inverse_transform(self._data['y_eval'])
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(1, mse, mae, rmse, mape))
def evaluate(self, sess):
global_step = sess.run(tf.train.get_or_create_global_step())
test_results = self.run_epoch_generator(
sess,
self.model,
self._data['eval_loader'].get_iterator(),
return_output=True,
training=False)
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
test_loss, y_preds = test_results['loss'], test_results['outputs']
utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss],
global_step=global_step)
y_preds = np.concatenate(y_preds, axis=0)
scaler = self._data['scaler']
predictions = []
y_truths = []
for horizon_i in range(self._data['y_eval'].shape[1]):
y_truth = scaler.inverse_transform(
self._data['y_eval'][:, horizon_i, :, 0])
y_truths.append(y_truth)
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred,
labels=y_truth,
null_val=0)
mae = metrics.masked_mae_np(preds=y_pred,
labels=y_truth,
null_val=0)
mape = metrics.masked_mape_np(preds=y_pred,
labels=y_truth,
null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred,
labels=y_truth,
null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(horizon_i + 1, mse, mae, rmse, mape))
utils.add_simple_summary(self._writer, [
'%s_%d' % (item, horizon_i + 1)
for item in ['metric/rmse', 'metric/mae', 'metric/mse']
], [rmse, mae, mse],
global_step=global_step)
outputs = {'predictions': predictions, 'groundtruth': y_truths}
return outputs
def _calculate_metrics(self, prediction_results, metrics_summary, scaler, runId, data_norm, n_metrics=4):
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
y_preds = prediction_results['y_preds']
y_preds = np.concatenate(y_preds, axis=0)
y_truths = prediction_results['y_truths']
y_truths = np.concatenate(y_truths, axis=0)
predictions = []
for horizon_i in range(self._horizon):
y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :])
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format(
horizon_i + 1, mse, mae, rmse, mape
)
)
metrics_summary[runId, horizon_i * n_metrics + 0] = mse
metrics_summary[runId, horizon_i * n_metrics + 1] = mae
metrics_summary[runId, horizon_i * n_metrics + 2] = rmse
metrics_summary[runId, horizon_i * n_metrics + 3] = mape
tm_pred = scaler.inverse_transform(prediction_results['tm_pred'])
g_truth = scaler.inverse_transform(data_norm[self._seq_len:-self._horizon])
m_indicator = prediction_results['m_indicator']
er = metrics.error_ratio(y_pred=tm_pred,
y_true=g_truth,
measured_matrix=m_indicator)
metrics_summary[runId, -1] = er
self._logger.info('ER: {}'.format(er))
self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(runId))
return metrics_summary
def main(config):
logger = config.get_logger('test')
graph_pkl_filename = 'data/sensor_graph/adj_mx_unix.pkl'
_, _, adj_mat = utils.load_graph_data(graph_pkl_filename)
data = utils.load_dataset(dataset_dir='data/METR-LA',
batch_size=config["arch"]["args"]["batch_size"],
test_batch_size=config["arch"]["args"]["batch_size"])
test_data_loader = data['test_loader']
scaler = data['scaler']
num_test_iteration= math.ceil(data['x_test'].shape[0] / config["arch"]["args"]["batch_size"])
# build model architecture
adj_arg = {"adj_mat": adj_mat}
model = config.initialize('arch', module_arch, **adj_arg)
logger.info(model)
logger.info('Loading checkpoint: {} ...'.format(config.resume))
checkpoint = torch.load(config.resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
# prepare model for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
total_loss = 0.0
y_preds = torch.FloatTensor([])
y_truths = data['y_test'] # (6850, 12, 207, 2)
y_truths = scaler.inverse_transform(y_truths)
predictions = []
groundtruth = list()
with torch.no_grad():
for i, (x, y) in tqdm(enumerate(test_data_loader.get_iterator()), total=num_test_iteration):
x = torch.FloatTensor(x).cuda()
y = torch.FloatTensor(y).cuda()
outputs = model(x, y, 0) # (seq_length+1, batch_size, num_nodes*output_dim) (13, 50, 207*1)
y_preds = torch.cat([y_preds, outputs], dim=1)
y_preds = torch.transpose(y_preds, 0, 1)
y_preds = y_preds.detach().numpy() # cast to numpy array
print("--------test results--------")
for horizon_i in range(y_truths.shape[1]):
y_truth = np.squeeze(y_truths[:, horizon_i, :, 0]) # (6850, 207)
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :]) # (6850, 207)
predictions.append(y_pred)
groundtruth.append(y_truth)
mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0)
mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
print(
"Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format(
horizon_i + 1, mae, mape, rmse
)
)
log = {"Horizon": horizon_i+1, "MAE": mae, "MAPE": mape, "RMSE": rmse}
logger.info(log)
outputs = {
'predictions': predictions,
'groundtruth': groundtruth
}
# serialize test data
np.savez_compressed('saved/results/dcrnn_predictions.npz', **outputs)
print('Predictions saved as {}.'.format('saved/results/dcrnn_predictions.npz'))
def predict_and_save_results_mstgcn(net, data_loader, data_target_tensor,
global_step, _mean, _std, params_path,
type):
'''
:param net: nn.Module
:param data_loader: torch.utils.data.utils.DataLoader
:param data_target_tensor: tensor
:param epoch: int
:param _mean: (1, 1, 3, 1)
:param _std: (1, 1, 3, 1)
:param params_path: the path for saving the results
:return:
'''
net.train(False) # ensure dropout layers are in test mode
for name, param in net.named_parameters():
print(name, param)
with torch.no_grad():
data_target_tensor = data_target_tensor.cpu().numpy()
loader_length = len(data_loader) # nb of batch
prediction = [] # 存储所有batch的output
input = [] # 存储所有batch的input
for batch_index, batch_data in enumerate(data_loader):
encoder_inputs, labels = batch_data
input.append(encoder_inputs[:, :,
0:1].cpu().numpy()) # (batch, T', 1)
outputs = net(encoder_inputs)
prediction.append(outputs.detach().cpu().numpy())
if batch_index % 100 == 0:
print('predicting data set batch %s / %s' %
(batch_index + 1, loader_length))
input = np.concatenate(input, 0)
input = re_normalization(input, _mean, _std)
prediction = np.concatenate(prediction, 0) # (batch, T', 1)
print('input:', input.shape)
print('prediction:', prediction.shape)
print('data_target_tensor:', data_target_tensor.shape)
output_filename = os.path.join(
params_path, 'output_epoch_%s_%s' % (global_step, type))
np.savez(output_filename,
input=input,
prediction=prediction,
data_target_tensor=data_target_tensor)
# 计算误差
excel_list = []
prediction_length = prediction.shape[2]
for i in range(prediction_length):
assert data_target_tensor.shape[0] == prediction.shape[0]
print('current epoch: %s, predict %s points' % (global_step, i))
mae = mean_absolute_error(data_target_tensor[:, :, i],
prediction[:, :, i])
rmse = mean_squared_error(data_target_tensor[:, :, i],
prediction[:, :, i])**0.5
mape = masked_mape_np(data_target_tensor[:, :, i],
prediction[:, :, i], 0)
print('MAE: %.2f' % (mae))
print('RMSE: %.2f' % (rmse))
print('MAPE: %.2f' % (mape))
excel_list.extend([mae, rmse, mape])
# print overall results
mae = mean_absolute_error(data_target_tensor.reshape(-1, 1),
prediction.reshape(-1, 1))
mse = mean_squared_error(data_target_tensor.reshape(-1, 1),
prediction.reshape(-1, 1))
rmse = mean_squared_error(data_target_tensor.reshape(-1, 1),
prediction.reshape(-1, 1))**0.5
mape = masked_mape_np(data_target_tensor.reshape(-1, 1),
prediction.reshape(-1, 1), 0)
print('all MAE: %.2f' % (mae))
print('all MSE: %.2f' % (mse))
print('all RMSE: %.2f' % (rmse))
print('all MAPE: %.2f' % (mape))
excel_list.extend([mae, rmse, mape])
print(excel_list)
def _test(self):
scaler = self._data['scaler']
results_summary = pd.DataFrame(index=range(self._run_times))
results_summary['No.'] = range(self._run_times)
n_metrics = 4
# Metrics: MSE, MAE, RMSE, MAPE, ER
metrics_summary = np.zeros(shape=(self._run_times,
self._horizon * n_metrics + 1))
for i in range(self._run_times):
print('|--- Running time: {}/{}'.format(i, self._run_times))
outputs = self._run_tm_prediction()
tm_pred, m_indicator, y_preds = outputs['tm_pred'], outputs[
'm_indicator'], outputs['y_preds']
y_preds = np.concatenate(y_preds, axis=0)
predictions = []
y_truths = outputs['y_truths']
y_truths = np.concatenate(y_truths, axis=0)
for horizon_i in range(self._horizon):
y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :])
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred,
labels=y_truth,
null_val=0)
mae = metrics.masked_mae_np(preds=y_pred,
labels=y_truth,
null_val=0)
mape = metrics.masked_mape_np(preds=y_pred,
labels=y_truth,
null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred,
labels=y_truth,
null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
.format(horizon_i + 1, mse, mae, rmse, mape))
metrics_summary[i, horizon_i * n_metrics + 0] = mse
metrics_summary[i, horizon_i * n_metrics + 1] = mae
metrics_summary[i, horizon_i * n_metrics + 2] = rmse
metrics_summary[i, horizon_i * n_metrics + 3] = mape
tm_pred = scaler.inverse_transform(tm_pred)
g_truth = scaler.inverse_transform(
self._data['test_data_norm'][self._seq_len:-self._horizon])
er = metrics.error_ratio(y_pred=tm_pred,
y_true=g_truth,
measured_matrix=m_indicator)
metrics_summary[i, -1] = er
self._save_results(g_truth=g_truth,
pred_tm=tm_pred,
m_indicator=m_indicator,
tag=str(i))
print('ER: {}'.format(er))
for horizon_i in range(self._horizon):
results_summary['mse_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0]
results_summary['mae_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1]
results_summary['rmse_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2]
results_summary['mape_{}'.format(
horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3]
results_summary['er'] = metrics_summary[:, -1]
results_summary.to_csv(self._log_dir + 'results_summary.csv',
index=False)
def _test(self, sess, **kwargs):
global_step = sess.run(tf.train.get_or_create_global_step())
results_summary = pd.DataFrame(index=range(self._run_times))
results_summary['No.'] = range(self._run_times)
n_metrics = 4
# Metrics: MSE, MAE, RMSE, MAPE, ER
metrics_summary = np.zeros(shape=(self._run_times, self._horizon * n_metrics + 1))
for i in range(self._run_times):
self._logger.info('|--- Run time: {}'.format(i))
# y_test = self._prepare_test_set()
test_results = self._run_tm_prediction(sess, model=self._test_model)
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
test_loss, y_preds = test_results['loss'], test_results['y_preds']
utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step)
y_preds = test_results['y_preds']
y_preds = np.concatenate(y_preds, axis=0)
y_truths = test_results['y_truths']
y_truths = np.concatenate(y_truths, axis=0)
scaler = self._data['scaler']
predictions = []
for horizon_i in range(self._horizon):
y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :, 0])
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0])
predictions.append(y_pred)
mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format(
horizon_i + 1, mse, mae, rmse, mape
)
)
metrics_summary[i, horizon_i * n_metrics + 0] = mse
metrics_summary[i, horizon_i * n_metrics + 1] = mae
metrics_summary[i, horizon_i * n_metrics + 2] = rmse
metrics_summary[i, horizon_i * n_metrics + 3] = mape
tm_pred = scaler.inverse_transform(test_results['tm_pred'])
g_truth = scaler.inverse_transform(self._data['test_data_norm'][self._seq_len:-self._horizon])
m_indicator = test_results['m_indicator']
er = error_ratio(y_pred=tm_pred,
y_true=g_truth,
measured_matrix=m_indicator)
metrics_summary[i, -1] = er
self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(i))
print('ER: {}'.format(er))
for horizon_i in range(self._horizon):
results_summary['mse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0]
results_summary['mae_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1]
results_summary['rmse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2]
results_summary['mape_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3]
results_summary['er'] = metrics_summary[:, -1]
results_summary.to_csv(self._log_dir + 'results_summary.csv', index=False)
return
def evaluate(model,
dataset,
dataset_type,
edge_index,
edge_attr,
device,
output_dim,
logger,
detail=True,
cfg=None,
format_result=False):
if detail:
logger.info('Evaluation_{}_Begin:'.format(dataset_type))
scaler = dataset['scaler']
y_preds = run_model(
model,
data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(),
edge_index=edge_index,
edge_attr=edge_attr,
device=device,
output_dim=output_dim)
y_preds = np.concatenate(y_preds, axis=0) # concat in batch_size dim.
mae_sum = 0
mape_sum = 0
rmse_sum = 0
mae_list = []
mape_list = []
rmse_list = []
mae_peak_list = []
mape_peak_list = []
rmse_peak_list = []
# horizon = dataset['y_{}'.format(dataset_type)].shape[1]
horizon = cfg['model']['horizon']
ytime = dataset['ytime_{}'.format(dataset_type)][:, 0]
ytime_pd = pd.to_datetime(ytime.flatten())
morning_peak = np.logical_and(
(ytime_pd.hour * 60 + ytime_pd.minute) >= 7 * 60 + 30,
(ytime_pd.hour * 60 + ytime_pd.minute) <= 8 * 60 + 30)
evening_peak = np.logical_and(
(ytime_pd.hour * 60 + ytime_pd.minute) >= 17 * 60 + 30,
(ytime_pd.hour * 60 + ytime_pd.minute) <= 18 * 60 + 30)
peak = np.logical_or(morning_peak, evening_peak)
for horizon_i in range(horizon):
y_truth = scaler.inverse_transform(
dataset['y_{}'.format(dataset_type)][:, horizon_i,
shtop72_index, :output_dim])
y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0], horizon_i,
shtop72_index, :output_dim])
mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn')
mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
mae_sum += mae
mape_sum += mape
rmse_sum += rmse
mae_list.append(mae)
mape_list.append(mape)
rmse_list.append(rmse)
# Peak
y_truth_peak = scaler.inverse_transform(
dataset['y_{}'.format(dataset_type)][peak,
horizon_i, :, :output_dim])
y_pred_peak = scaler.inverse_transform(
y_preds[:y_truth.shape[0], horizon_i, :, :output_dim])
y_pred_peak = y_pred_peak[peak]
mae_peak = metrics.masked_mae_np(y_pred_peak,
y_truth_peak,
null_val=0,
mode='dcrnn')
mape_peak = metrics.masked_mape_np(y_pred_peak,
y_truth_peak,
null_val=0)
rmse_peak = metrics.masked_rmse_np(y_pred_peak,
y_truth_peak,
null_val=0)
mae_peak_list.append(mae_peak)
mape_peak_list.append(mape_peak)
rmse_peak_list.append(rmse_peak)
msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}"
if detail:
logger.info(msg.format(horizon_i + 1, mae, mape, rmse))
if detail:
logger.info('Evaluation_{}_End:'.format(dataset_type))
if format_result:
print('-' * 40 + ' 1/4 ' + '-' * 40)
for i in range(len(mape_list)):
print('{:.2f}'.format(mae_list[i]))
print('{:.2f}%'.format(mape_list[i] * 100))
print('{:.2f}'.format(rmse_list[i]))
print()
print('-' * 40 + ' Peak ' + '-' * 40)
for i in range(len(mape_list)):
print('{:.2f}'.format(mae_peak_list[i]))
print('{:.2f}%'.format(mape_peak_list[i] * 100))
print('{:.2f}'.format(rmse_peak_list[i]))
print()
else:
return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
def evaluate(self, sess, **kwargs):
global_step = sess.run(tf.train.get_or_create_global_step())
test_results = self.run_epoch_generator(
sess,
self._test_model,
self._data['test_loader'].get_iterator(),
return_output=True,
training=False,
return_ground_truth=True,
task='test')
test_loss = test_results['loss']
# utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss],
# global_step=global_step)
utils.add_summary_dict(self._writer, {
'loss/test_loss': test_loss,
},
global_step=global_step)
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
y_preds = np.concatenate(test_results['outputs'], axis=0)
scaler = self._data['scaler']
y_truths = np.concatenate(test_results['ground_truths'], axis=0)
assert y_preds.shape[:3] == y_truths.shape[:3], \
'NOT {} == {}'.format(y_preds.shape[:3], y_truths.shape[:3])
y_preds_original = []
y_truths_original = []
min_output_value = self._data_kwargs.get('min_output_value')
max_output_value = self._data_kwargs.get('max_output_value')
clip = (min_output_value is not None) or (max_output_value is not None)
if clip:
self._kwargs.logger.info('The output values are clipped to range: [{}, {}]'.\
format(min_output_value, max_output_value))
for horizon_i in range(y_truths.shape[1]):
y_pred_original = scaler.inverse_transform(y_preds[:, horizon_i, :,
0])
if clip:
y_pred_original = \
np.clip(y_pred_original, min_output_value, max_output_value,
out=y_pred_original)
y_preds_original.append(y_pred_original)
self._kwargs.logger.info(
stat_str(y_pred_original, horizon_i, 'pred'))
y_truth_original = scaler.inverse_transform(y_truths[:,
horizon_i, :,
0])
y_truths_original.append(y_truth_original)
if not np.all(np.isnan(y_truth_original)):
self._kwargs.logger.info(
stat_str(y_truth_original, horizon_i, 'true'))
for null_val in [0, np.nan]:
desc = r'0 excl.' if null_val == 0 else 'any '
rmse = metrics.masked_rmse_np(y_pred_original,
y_truth_original,
null_val=null_val)
r2 = metrics.masked_r2_np(y_pred_original,
y_truth_original,
null_val=null_val)
mae = metrics.masked_mae_np(y_pred_original,
y_truth_original,
null_val=null_val)
mape = metrics.masked_mape_np(y_pred_original, y_truth_original, null_val=null_val) \
if null_val == 0 else np.nan
self._kwargs.logger.info(
"T+{:02d}|{}|RMSE: {:8.5f}|R2: {:8.4f}|MAE: {:5.2f}|MAPE: {:5.3f}|".\
format(horizon_i + 1, desc, rmse, r2, mae, mape,)
)
# utils.add_simple_summary(self._writer,
# ['%s_%d' % (item, horizon_i + 1) for item in
# ['metric/rmse', 'metric/mape', 'metric/mae']],
# [rmse, mape, mae],
# global_step=global_step)
utils.add_summary_dict(
self._writer, {
'metric/rmse_horizon{}'.format(horizon_i + 1): rmse,
'metric/mape_horizon{}'.format(horizon_i + 1): mape,
'metric/mae_horizon{}'.format(horizon_i + 1): mae,
'metric/r2_horizon{}'.format(horizon_i + 1): r2,
},
global_step=global_step)
outputs = {
'predictions': y_preds_original,
'groundtruth': y_truths_original
}
return outputs
def test_mape():
from lib.metrics import masked_mape_np
a = np.random.uniform(size=(2, 2))
b = np.random.uniform(size=(2, 2))
mape = masked_mape_np(a, b, 0)
assert type(mape) == np.float64
def _train(self,
sess,
base_lr,
epoch,
steps,
patience=50,
epochs=100,
min_learning_rate=2e-6,
lr_decay_ratio=0.1,
save_model=1,
test_every_n_epochs=10,
**train_kwargs):
history = []
min_val_loss = float('inf')
wait = 0
max_to_keep = train_kwargs.get('max_to_keep', 100)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep)
model_filename = train_kwargs.get('model_filename')
output_file = train_kwargs.get('preds_file')
gt_file = train_kwargs.get('groundtruth_file')
if model_filename is not None:
saver.restore(sess, model_filename)
self._epoch = epoch + 1
else:
sess.run(tf.global_variables_initializer())
self._logger.info('Start training ...')
while self._epoch <= epochs:
# Learning rate schedule.
new_lr = max(
min_learning_rate,
base_lr *
(lr_decay_ratio**np.sum(self._epoch >= np.array(steps))))
self.set_lr(sess=sess, lr=new_lr)
start_time = time.time()
train_results = self.run_epoch_generator(
sess,
self._train_model,
self._data['train_loader'].get_iterator(),
training=True,
writer=self._writer)
train_loss, train_mae, train_reg = train_results[
'loss'], train_results['mae'], train_results['reg']
#print ('reg loss is:', train_reg)
if train_loss > 1e5:
self._logger.warning('Gradient explosion detected. Ending...')
break
global_step = sess.run(tf.train.get_or_create_global_step())
# Compute validation error.
val_results = self.run_epoch_generator(
sess,
self._test_model,
self._data['val_loader'].get_iterator(),
training=False)
val_loss, val_mae = np.asscalar(val_results['loss']), np.asscalar(
val_results['mae'])
utils.add_simple_summary(self._writer, [
'loss/train_loss', 'metric/train_mae', 'loss/val_loss',
'metric/val_mae'
], [train_loss, train_mae, val_loss, val_mae],
global_step=global_step)
end_time = time.time()
message = 'Epoch [{}/{}] ({}) train_mae: {:.4f}, val_mae: {:.4f} lr:{:.6f} {:.1f}s'.format(
self._epoch, epochs, global_step, train_mae, val_mae, new_lr,
(end_time - start_time))
self._logger.info(message)
stt = time.time()
outputs = self.evaluate(sess)
#print (outputs['groundtruth'].shape)
test_gdt = outputs['groundtruth'][:, :, :, 0]
test_y = outputs['observed'][:, :, :, 0]
best_pred = outputs['predictions'][:test_gdt.shape[0], :, :, 0]
test_gdt = test_gdt.reshape(
test_gdt.shape[0], -1, order='F'
) # record the corresponding steps for each node first, then node by node
test_y = test_y.reshape(test_gdt.shape[0], -1, order='F')
best_pred = best_pred.reshape(test_gdt.shape[0], -1, order='F')
scaler = self._data['scaler']
print('Test running time: %fs' % (time.time() - stt))
# --------
# best_pred = scaler.inverse_transform(best_pred)
# # np.save('best_pred.npy', best_pred)
# mape = metrics.masked_mape_np(best_pred, test_y, test_gdt, null_val=0)
# rmse = metrics.masked_rmse_np(best_pred, test_y, test_gdt, null_val=0)
# self._logger.info(
# 'Overall Test MAPE %.4f, RMSE %.4f' % (mape, rmse))
# --------
if val_loss <= min_val_loss:
wait = 0
if save_model > 0:
model_filename = self.save(sess, val_loss)
self._logger.info(
'Val loss decrease from %.4f to %.4f, saving to %s' %
(min_val_loss, val_loss, model_filename))
min_val_loss = val_loss
test_y = scaler.inverse_transform(test_y)
best_pred = scaler.inverse_transform(best_pred)
# np.save('best_pred.npy', best_pred)
mape = metrics.masked_mape_np(best_pred,
test_y,
test_gdt,
null_val=0)
rmse = metrics.masked_rmse_np(best_pred,
test_y,
test_gdt,
null_val=0)
self._logger.info('Overall Test MAPE %.4f, RMSE %.4f' %
(mape, rmse))
print(best_pred.shape)
#np.savetxt(output_file, best_pred, delimiter = ',')
#np.savetxt(gt_file, test_gdt, delimiter = ',')
else:
wait += 1
if wait > patience:
self._logger.warning('Early stopping at epoch: %d' %
self._epoch)
break
# test_y = scaler.inverse_transform(test_y)
# best_pred = scaler.inverse_transform(best_pred)
# mape = metrics.masked_mape_np(best_pred, test_y, test_gdt, null_val=0)
# rmse = metrics.masked_rmse_np(best_pred, test_y, test_gdt, null_val=0)
# self._logger.info(
# 'Overall Test MAPE %.4f, RMSE %.4f' % (mape, rmse))
history.append(val_mae)
# Increases epoch.
self._epoch += 1
sys.stdout.flush()
return np.min(history)