def train(self): training_history = self.model.fit( x=self._data['x_train'], y=self._data['y_train'], batch_size=self._batch_size, epochs=self._epochs, callbacks=self.callbacks_list, validation_data=(self._data['x_val'], self._data['y_val']), shuffle=True, verbose=2) if training_history is not None: self._plot_training_history(training_history) self._save_model_history(training_history) config = dict(self._kwargs) config_filename = 'config_lstm.yaml' config['train']['log_dir'] = self._log_dir with open(os.path.join(self._log_dir, config_filename), 'w') as f: yaml.dump(config, f, default_flow_style=False) # evaluate scaler = self._data['scaler'] x_eval = self._data['x_eval'] y_truth = self._data['y_eval'] y_pred = self.model.predict(x_eval) y_pred = scaler.inverse_transform(y_pred) y_truth = scaler.inverse_transform(y_truth) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format( 1, mse, mape, rmse))
def evaluate(self): scaler = self._data['scaler'] y_pred_1, y_pred_2 = self.model.predict(self._data['x_eval']) y_pred_1 = scaler.inverse_transform(y_pred_1) y_truth_1 = scaler.inverse_transform(self._data['y_eval_1']) y_truth_2 = scaler.inverse_transform(self._data['y_eval_2']) mse = metrics.masked_mse_np(preds=y_pred_1, labels=y_truth_1, null_val=0) mae = metrics.masked_mae_np(preds=y_pred_1, labels=y_truth_1, null_val=0) mape = metrics.masked_mape_np(preds=y_pred_1, labels=y_truth_1, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred_1, labels=y_truth_1, null_val=0) self._logger.info( " Forward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(mse, mae, rmse, mape)) mse_2 = metrics.masked_mse_np(preds=y_pred_2, labels=y_truth_2, null_val=0) mae_2 = metrics.masked_mae_np(preds=y_pred_2, labels=y_truth_2, null_val=0) mape_2 = metrics.masked_mape_np(preds=y_pred_2, labels=y_truth_2, null_val=0) rmse_2 = metrics.masked_rmse_np(preds=y_pred_2, labels=y_truth_2, null_val=0) self._logger.info( "Backward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(mse_2, mae_2, rmse_2, mape_2))
def evaluate(self): scaler = self._data['scaler'] y_pred = self.model.predict(self._data['x_eval']) y_pred = scaler.inverse_transform(y_pred) y_truth = scaler.inverse_transform(self._data['y_eval']) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(1, mse, mae, rmse, mape))
def evaluate(self, sess): global_step = sess.run(tf.train.get_or_create_global_step()) test_results = self.run_epoch_generator( sess, self.model, self._data['eval_loader'].get_iterator(), return_output=True, training=False) # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) test_loss, y_preds = test_results['loss'], test_results['outputs'] utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step) y_preds = np.concatenate(y_preds, axis=0) scaler = self._data['scaler'] predictions = [] y_truths = [] for horizon_i in range(self._data['y_eval'].shape[1]): y_truth = scaler.inverse_transform( self._data['y_eval'][:, horizon_i, :, 0]) y_truths.append(y_truth) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(horizon_i + 1, mse, mae, rmse, mape)) utils.add_simple_summary(self._writer, [ '%s_%d' % (item, horizon_i + 1) for item in ['metric/rmse', 'metric/mae', 'metric/mse'] ], [rmse, mae, mse], global_step=global_step) outputs = {'predictions': predictions, 'groundtruth': y_truths} return outputs
def _calculate_metrics(self, prediction_results, metrics_summary, scaler, runId, data_norm, n_metrics=4): # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) y_preds = prediction_results['y_preds'] y_preds = np.concatenate(y_preds, axis=0) y_truths = prediction_results['y_truths'] y_truths = np.concatenate(y_truths, axis=0) predictions = [] for horizon_i in range(self._horizon): y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :]) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format( horizon_i + 1, mse, mae, rmse, mape ) ) metrics_summary[runId, horizon_i * n_metrics + 0] = mse metrics_summary[runId, horizon_i * n_metrics + 1] = mae metrics_summary[runId, horizon_i * n_metrics + 2] = rmse metrics_summary[runId, horizon_i * n_metrics + 3] = mape tm_pred = scaler.inverse_transform(prediction_results['tm_pred']) g_truth = scaler.inverse_transform(data_norm[self._seq_len:-self._horizon]) m_indicator = prediction_results['m_indicator'] er = metrics.error_ratio(y_pred=tm_pred, y_true=g_truth, measured_matrix=m_indicator) metrics_summary[runId, -1] = er self._logger.info('ER: {}'.format(er)) self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(runId)) return metrics_summary
def _test(self): scaler = self._data['scaler'] results_summary = pd.DataFrame(index=range(self._run_times)) results_summary['No.'] = range(self._run_times) n_metrics = 4 # Metrics: MSE, MAE, RMSE, MAPE, ER metrics_summary = np.zeros(shape=(self._run_times, self._horizon * n_metrics + 1)) for i in range(self._run_times): print('|--- Running time: {}/{}'.format(i, self._run_times)) outputs = self._run_tm_prediction() tm_pred, m_indicator, y_preds = outputs['tm_pred'], outputs[ 'm_indicator'], outputs['y_preds'] y_preds = np.concatenate(y_preds, axis=0) predictions = [] y_truths = outputs['y_truths'] y_truths = np.concatenate(y_truths, axis=0) for horizon_i in range(self._horizon): y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :]) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(horizon_i + 1, mse, mae, rmse, mape)) metrics_summary[i, horizon_i * n_metrics + 0] = mse metrics_summary[i, horizon_i * n_metrics + 1] = mae metrics_summary[i, horizon_i * n_metrics + 2] = rmse metrics_summary[i, horizon_i * n_metrics + 3] = mape tm_pred = scaler.inverse_transform(tm_pred) g_truth = scaler.inverse_transform( self._data['test_data_norm'][self._seq_len:-self._horizon]) er = metrics.error_ratio(y_pred=tm_pred, y_true=g_truth, measured_matrix=m_indicator) metrics_summary[i, -1] = er self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(i)) print('ER: {}'.format(er)) for horizon_i in range(self._horizon): results_summary['mse_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0] results_summary['mae_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1] results_summary['rmse_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2] results_summary['mape_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3] results_summary['er'] = metrics_summary[:, -1] results_summary.to_csv(self._log_dir + 'results_summary.csv', index=False)
def _test(self, sess, **kwargs): global_step = sess.run(tf.train.get_or_create_global_step()) results_summary = pd.DataFrame(index=range(self._run_times)) results_summary['No.'] = range(self._run_times) n_metrics = 4 # Metrics: MSE, MAE, RMSE, MAPE, ER metrics_summary = np.zeros(shape=(self._run_times, self._horizon * n_metrics + 1)) for i in range(self._run_times): self._logger.info('|--- Run time: {}'.format(i)) # y_test = self._prepare_test_set() test_results = self._run_tm_prediction(sess, model=self._test_model) # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) test_loss, y_preds = test_results['loss'], test_results['y_preds'] utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step) y_preds = test_results['y_preds'] y_preds = np.concatenate(y_preds, axis=0) y_truths = test_results['y_truths'] y_truths = np.concatenate(y_truths, axis=0) scaler = self._data['scaler'] predictions = [] for horizon_i in range(self._horizon): y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :, 0]) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format( horizon_i + 1, mse, mae, rmse, mape ) ) metrics_summary[i, horizon_i * n_metrics + 0] = mse metrics_summary[i, horizon_i * n_metrics + 1] = mae metrics_summary[i, horizon_i * n_metrics + 2] = rmse metrics_summary[i, horizon_i * n_metrics + 3] = mape tm_pred = scaler.inverse_transform(test_results['tm_pred']) g_truth = scaler.inverse_transform(self._data['test_data_norm'][self._seq_len:-self._horizon]) m_indicator = test_results['m_indicator'] er = error_ratio(y_pred=tm_pred, y_true=g_truth, measured_matrix=m_indicator) metrics_summary[i, -1] = er self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(i)) print('ER: {}'.format(er)) for horizon_i in range(self._horizon): results_summary['mse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0] results_summary['mae_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1] results_summary['rmse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2] results_summary['mape_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3] results_summary['er'] = metrics_summary[:, -1] results_summary.to_csv(self._log_dir + 'results_summary.csv', index=False) return