def test_masked_mape_np_nan(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([
[np.nan, np.nan],
[np.nan, 3]
], dtype=np.float32)
mape = metrics_np.masked_mape_np(preds=preds, labels=labels)
self.assertAlmostEqual(1 / 3., mape, delta=1e-5)
def test_masked_mape_np_all_zero(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([
[0, 0],
[0, 0]
], dtype=np.float32)
mape = metrics_np.masked_mape_np(preds=preds, labels=labels, null_val=0)
self.assertEqual(0., mape)
def test_masked_mape_np_all_nan(self):
preds = np.array([
[1, 2],
[3, 4],
], dtype=np.float32)
labels = np.array([
[np.nan, np.nan],
[np.nan, np.nan]
], dtype=np.float32)
mape = metrics_np.masked_mape_np(preds=preds, labels=labels)
self.assertEqual(0., mape)
def test_masked_mape_np2(self):
preds = np.array([
[1, 2, 2],
[3, 4, 5],
], dtype=np.float32)
labels = np.array([
[1, 2, 2],
[3, 4, 4]
], dtype=np.float32)
mape = metrics_np.masked_mape_np(preds=preds, labels=labels, null_val=4)
self.assertEqual(0., mape)
def test_masked_mape_np(self):
preds = np.array([
[1, 2, 2],
[3, 4, 5],
], dtype=np.float32)
labels = np.array([
[1, 2, 2],
[3, 4, 4]
], dtype=np.float32)
mape = metrics_np.masked_mape_np(preds=preds, labels=labels)
self.assertAlmostEqual(1 / 24.0, mape, delta=1e-5)
def evaluate(self, sess, **kwargs):
global_step = sess.run(tf.train.get_or_create_global_step())
test_results = self.run_epoch_generator(
sess,
self._test_model,
self._data['test_loader'].get_iterator(),
return_output=True,
training=False)
y_truths = []
for x, y in self._data['test_loader'].get_iterator():
for row in y:
y_truths.append(row)
y_truths = np.array(y_truths)
# y_preds: a list of (batch_size, horizon, num_nodes, output_dim)
test_loss, y_preds = test_results['loss'], test_results['outputs']
add_simple_summary(self._writer, ['loss/test_loss'], [test_loss],
global_step=global_step)
y_preds = np.concatenate(y_preds, axis=0)
scaler = self._data['scaler']
predictions = []
# y_truths = []
for horizon_i in range(self._data['y_test'].shape[1]):
# y_truth = scaler.inverse_transform(self._data['y_test'][:, horizon_i, :, 0])
# y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0], horizon_i, :, 0])
# y_truth = self._data['y_test'][:, horizon_i, :, 0] # Not scaled by our dataloader
y_truth = y_truths[:y_preds.shape[0], horizon_i, :, 0]
y_pred = y_preds[:, horizon_i, :,
0] # Already scaled back by self.run_epoch_generator
# y_truths.append(y_truth)
predictions.append(y_pred)
try:
mae = metrics_np.masked_mae_np(y_pred, y_truth, null_val=0)
mape = metrics_np.masked_mape_np(y_pred, y_truth, null_val=0)
rmse = metrics_np.masked_rmse_np(y_pred, y_truth, null_val=0)
self._logger.info(
"Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".
format(horizon_i + 1, mae, mape, rmse))
add_simple_summary(self._writer, [
'%s_%d' % (item, horizon_i + 1)
for item in ['metric/rmse', 'metric/mape', 'metric/mae']
], [rmse, mape, mae],
global_step=global_step)
except:
pass
outputs = {'predictions': predictions, 'groundtruth': y_truths}
return outputs
def eval_historical_average(traffic_reading_df, period):
y_predict, y_test = historical_average_predict(traffic_reading_df,
period=period,
test_ratio=0.2)
rmse = masked_rmse_np(preds=y_predict.as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mape = masked_mape_np(preds=y_predict.as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mae = masked_mae_np(preds=y_predict.as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
logger.info('Historical Average')
logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
for horizon in [1, 3, 6, 12]:
line = 'HA\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def eval_static(traffic_reading_df):
logger.info('Static')
horizons = [1, 3, 6, 12]
logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
for horizon in horizons:
y_predict, y_test = static_predict(traffic_reading_df,
n_forward=horizon,
test_ratio=0.2)
rmse = masked_rmse_np(preds=y_predict.as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mape = masked_mape_np(preds=y_predict.as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mae = masked_mae_np(preds=y_predict.as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
line = 'Static\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100,
mae)
logger.info(line)
def eval_var(traffic_reading_df, n_lags=3):
n_forwards = [1, 3, 6, 12]
y_predicts, y_test = var_predict(traffic_reading_df,
n_forwards=n_forwards,
n_lags=n_lags,
test_ratio=0.2)
logger.info('VAR (lag=%d)' % n_lags)
logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE')
for i, horizon in enumerate(n_forwards):
rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mape = masked_mape_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mae = masked_mae_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)