def _set_measured_flow(self, rnn_input, pred_forward, m_indicator): """ :param rnn_input: shape(#seq_len, #nflows) :param pred_forward: shape(#seq_len, #nflows) :param m_indicator: shape(#seq_len, #nflows) :return: """ n_flows = rnn_input.shape[0] fw_losses = [] for flow_id in range(m_indicator.shape[1]): idx_fw = m_indicator[1:, flow_id] # fw_losses.append(error_ratio(y_true=rnn_input[1:, flow_id][idx_fw == 1.0], # y_pred=pred_forward[:-1, flow_id][idx_fw == 1.0], # measured_matrix=np.zeros(idx_fw[idx_fw == 1.0].shape))) fw_losses.append(metrics.masked_mae_np(preds=pred_forward[:-1, flow_id][idx_fw == 1.0], labels=rnn_input[1:, flow_id][idx_fw == 1.0])) fw_losses = np.array(fw_losses) fw_losses[fw_losses == 0.] = np.max(fw_losses) w = self._calculate_flows_weights(fw_losses=fw_losses, m_indicator=m_indicator) sampling = np.zeros(shape=n_flows) m = int(self._mon_ratio * n_flows) w = w.flatten() sorted_idx_w = np.argsort(w) sampling[sorted_idx_w[:m]] = 1 return sampling
def eval_var(traffic_reading_df, n_lags=3): n_forwards = [1, 2] y_predicts, y_test = var_predict(traffic_reading_df, n_forwards=n_forwards, n_lags=n_lags, test_ratio=0.2) #print("y_predicts type:" + types(y_predicts)) print("type===========") print(type(y_predicts[1].as_matrix())) #df_list = pd.DataFrame(y_predicts) #df_list.to_csv("./hello.csv", sep=',', index=False) logger.info('VAR (lag=%d)' % n_lags) logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE') for i, horizon in enumerate(n_forwards): rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) mape = masked_mape_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) mae = masked_mae_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) logger.info(line)
def evaluate(model, dataset, dataset_type, edge_index, edge_attr, device, output_dim, logger, detail=True, cfg=None, format_result=False): if detail: logger.info('Evaluation_{}_Begin:'.format(dataset_type)) scaler = dataset['scaler'] y_preds = run_model( model, data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(), edge_index=edge_index, edge_attr=edge_attr, device=device, output_dim=output_dim) y_preds = np.concatenate(y_preds, axis=0) # concat in batch_size dim. mae_list = [] mape_list = [] rmse_list = [] mae_sum = 0 mape_sum = 0 rmse_sum = 0 # horizon = dataset['y_{}'.format(dataset_type)].shape[1] horizon = cfg['model']['horizon'] for horizon_i in range(horizon): y_truth = scaler.inverse_transform( dataset['y_{}'.format(dataset_type)][:, horizon_i, :, :output_dim]) y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0], horizon_i, :, :output_dim]) mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn') mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0) rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0) mae_sum += mae mape_sum += mape rmse_sum += rmse mae_list.append(mae) mape_list.append(mape) rmse_list.append(rmse) msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}" if detail: logger.info(msg.format(horizon_i + 1, mae, mape, rmse)) if detail: logger.info('Evaluation_{}_End:'.format(dataset_type)) if format_result: for i in range(len(mape_list)): print('{:.2f}'.format(mae_list[i])) print('{:.2f}%'.format(mape_list[i] * 100)) print('{:.2f}'.format(rmse_list[i])) print() else: return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
def eval_historical_average(traffic_reading_df, period): y_predict, y_test = historical_average_predict(traffic_reading_df, period=period, test_ratio=0.2) rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) logger.info('Historical Average') logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE'])) for horizon in [1, 3, 6, 12]: line = 'HA\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) logger.info(line)
def eval_static(traffic_reading_df): logger.info('Static') horizons = [1, 3, 6, 12] logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE'])) for horizon in horizons: y_predict, y_test = static_predict(traffic_reading_df, n_forward=horizon, test_ratio=0.2) rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) line = 'Static\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) logger.info(line)
def eval_var(traffic_reading_df, n_lags=3): n_forwards = [1, 3, 6, 12] y_predicts, y_test = var_predict(traffic_reading_df, n_forwards=n_forwards, n_lags=n_lags, test_ratio=0.2) logger.info('VAR (lag=%d)' % n_lags) logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE') for i, horizon in enumerate(n_forwards): rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) mape = masked_mape_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) mae = masked_mae_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) logger.info(line)
def evaluate(self): scaler = self._data['scaler'] y_pred_1, y_pred_2 = self.model.predict(self._data['x_eval']) y_pred_1 = scaler.inverse_transform(y_pred_1) y_truth_1 = scaler.inverse_transform(self._data['y_eval_1']) y_truth_2 = scaler.inverse_transform(self._data['y_eval_2']) mse = metrics.masked_mse_np(preds=y_pred_1, labels=y_truth_1, null_val=0) mae = metrics.masked_mae_np(preds=y_pred_1, labels=y_truth_1, null_val=0) mape = metrics.masked_mape_np(preds=y_pred_1, labels=y_truth_1, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred_1, labels=y_truth_1, null_val=0) self._logger.info( " Forward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(mse, mae, rmse, mape)) mse_2 = metrics.masked_mse_np(preds=y_pred_2, labels=y_truth_2, null_val=0) mae_2 = metrics.masked_mae_np(preds=y_pred_2, labels=y_truth_2, null_val=0) mape_2 = metrics.masked_mape_np(preds=y_pred_2, labels=y_truth_2, null_val=0) rmse_2 = metrics.masked_rmse_np(preds=y_pred_2, labels=y_truth_2, null_val=0) self._logger.info( "Backward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(mse_2, mae_2, rmse_2, mape_2))
def evaluate(self): scaler = self._data['scaler'] y_pred = self.model.predict(self._data['x_eval']) y_pred = scaler.inverse_transform(y_pred) y_truth = scaler.inverse_transform(self._data['y_eval']) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(1, mse, mae, rmse, mape))
def evaluate(self, sess): global_step = sess.run(tf.train.get_or_create_global_step()) test_results = self.run_epoch_generator( sess, self.model, self._data['eval_loader'].get_iterator(), return_output=True, training=False) # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) test_loss, y_preds = test_results['loss'], test_results['outputs'] utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step) y_preds = np.concatenate(y_preds, axis=0) scaler = self._data['scaler'] predictions = [] y_truths = [] for horizon_i in range(self._data['y_eval'].shape[1]): y_truth = scaler.inverse_transform( self._data['y_eval'][:, horizon_i, :, 0]) y_truths.append(y_truth) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(horizon_i + 1, mse, mae, rmse, mape)) utils.add_simple_summary(self._writer, [ '%s_%d' % (item, horizon_i + 1) for item in ['metric/rmse', 'metric/mae', 'metric/mse'] ], [rmse, mae, mse], global_step=global_step) outputs = {'predictions': predictions, 'groundtruth': y_truths} return outputs
def _calculate_metrics(self, prediction_results, metrics_summary, scaler, runId, data_norm, n_metrics=4): # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) y_preds = prediction_results['y_preds'] y_preds = np.concatenate(y_preds, axis=0) y_truths = prediction_results['y_truths'] y_truths = np.concatenate(y_truths, axis=0) predictions = [] for horizon_i in range(self._horizon): y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :]) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format( horizon_i + 1, mse, mae, rmse, mape ) ) metrics_summary[runId, horizon_i * n_metrics + 0] = mse metrics_summary[runId, horizon_i * n_metrics + 1] = mae metrics_summary[runId, horizon_i * n_metrics + 2] = rmse metrics_summary[runId, horizon_i * n_metrics + 3] = mape tm_pred = scaler.inverse_transform(prediction_results['tm_pred']) g_truth = scaler.inverse_transform(data_norm[self._seq_len:-self._horizon]) m_indicator = prediction_results['m_indicator'] er = metrics.error_ratio(y_pred=tm_pred, y_true=g_truth, measured_matrix=m_indicator) metrics_summary[runId, -1] = er self._logger.info('ER: {}'.format(er)) self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(runId)) return metrics_summary
def _test(self): scaler = self._data['scaler'] results_summary = pd.DataFrame(index=range(self._run_times)) results_summary['No.'] = range(self._run_times) n_metrics = 4 # Metrics: MSE, MAE, RMSE, MAPE, ER metrics_summary = np.zeros(shape=(self._run_times, self._horizon * n_metrics + 1)) for i in range(self._run_times): print('|--- Running time: {}/{}'.format(i, self._run_times)) outputs = self._run_tm_prediction() tm_pred, m_indicator, y_preds = outputs['tm_pred'], outputs[ 'm_indicator'], outputs['y_preds'] y_preds = np.concatenate(y_preds, axis=0) predictions = [] y_truths = outputs['y_truths'] y_truths = np.concatenate(y_truths, axis=0) for horizon_i in range(self._horizon): y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :]) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}" .format(horizon_i + 1, mse, mae, rmse, mape)) metrics_summary[i, horizon_i * n_metrics + 0] = mse metrics_summary[i, horizon_i * n_metrics + 1] = mae metrics_summary[i, horizon_i * n_metrics + 2] = rmse metrics_summary[i, horizon_i * n_metrics + 3] = mape tm_pred = scaler.inverse_transform(tm_pred) g_truth = scaler.inverse_transform( self._data['test_data_norm'][self._seq_len:-self._horizon]) er = metrics.error_ratio(y_pred=tm_pred, y_true=g_truth, measured_matrix=m_indicator) metrics_summary[i, -1] = er self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(i)) print('ER: {}'.format(er)) for horizon_i in range(self._horizon): results_summary['mse_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0] results_summary['mae_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1] results_summary['rmse_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2] results_summary['mape_{}'.format( horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3] results_summary['er'] = metrics_summary[:, -1] results_summary.to_csv(self._log_dir + 'results_summary.csv', index=False)
def _test(self, sess, **kwargs): global_step = sess.run(tf.train.get_or_create_global_step()) results_summary = pd.DataFrame(index=range(self._run_times)) results_summary['No.'] = range(self._run_times) n_metrics = 4 # Metrics: MSE, MAE, RMSE, MAPE, ER metrics_summary = np.zeros(shape=(self._run_times, self._horizon * n_metrics + 1)) for i in range(self._run_times): self._logger.info('|--- Run time: {}'.format(i)) # y_test = self._prepare_test_set() test_results = self._run_tm_prediction(sess, model=self._test_model) # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) test_loss, y_preds = test_results['loss'], test_results['y_preds'] utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step) y_preds = test_results['y_preds'] y_preds = np.concatenate(y_preds, axis=0) y_truths = test_results['y_truths'] y_truths = np.concatenate(y_truths, axis=0) scaler = self._data['scaler'] predictions = [] for horizon_i in range(self._horizon): y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :, 0]) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0]) predictions.append(y_pred) mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0) mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0) mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0) rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0) self._logger.info( "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format( horizon_i + 1, mse, mae, rmse, mape ) ) metrics_summary[i, horizon_i * n_metrics + 0] = mse metrics_summary[i, horizon_i * n_metrics + 1] = mae metrics_summary[i, horizon_i * n_metrics + 2] = rmse metrics_summary[i, horizon_i * n_metrics + 3] = mape tm_pred = scaler.inverse_transform(test_results['tm_pred']) g_truth = scaler.inverse_transform(self._data['test_data_norm'][self._seq_len:-self._horizon]) m_indicator = test_results['m_indicator'] er = error_ratio(y_pred=tm_pred, y_true=g_truth, measured_matrix=m_indicator) metrics_summary[i, -1] = er self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(i)) print('ER: {}'.format(er)) for horizon_i in range(self._horizon): results_summary['mse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0] results_summary['mae_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1] results_summary['rmse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2] results_summary['mape_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3] results_summary['er'] = metrics_summary[:, -1] results_summary.to_csv(self._log_dir + 'results_summary.csv', index=False) return
def evaluate(model, dataset, dataset_type, edge_index, edge_attr, device, output_dim, logger, detail=True, cfg=None, format_result=False): if detail: logger.info('Evaluation_{}_Begin:'.format(dataset_type)) scaler = dataset['scaler'] y_preds = run_model( model, data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(), edge_index=edge_index, edge_attr=edge_attr, device=device, output_dim=output_dim) y_preds = np.concatenate(y_preds, axis=0) # concat in batch_size dim. mae_sum = 0 mape_sum = 0 rmse_sum = 0 mae_list = [] mape_list = [] rmse_list = [] mae_peak_list = [] mape_peak_list = [] rmse_peak_list = [] # horizon = dataset['y_{}'.format(dataset_type)].shape[1] horizon = cfg['model']['horizon'] ytime = dataset['ytime_{}'.format(dataset_type)][:, 0] ytime_pd = pd.to_datetime(ytime.flatten()) morning_peak = np.logical_and( (ytime_pd.hour * 60 + ytime_pd.minute) >= 7 * 60 + 30, (ytime_pd.hour * 60 + ytime_pd.minute) <= 8 * 60 + 30) evening_peak = np.logical_and( (ytime_pd.hour * 60 + ytime_pd.minute) >= 17 * 60 + 30, (ytime_pd.hour * 60 + ytime_pd.minute) <= 18 * 60 + 30) peak = np.logical_or(morning_peak, evening_peak) for horizon_i in range(horizon): y_truth = scaler.inverse_transform( dataset['y_{}'.format(dataset_type)][:, horizon_i, shtop72_index, :output_dim]) y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0], horizon_i, shtop72_index, :output_dim]) mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn') mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0) rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0) mae_sum += mae mape_sum += mape rmse_sum += rmse mae_list.append(mae) mape_list.append(mape) rmse_list.append(rmse) # Peak y_truth_peak = scaler.inverse_transform( dataset['y_{}'.format(dataset_type)][peak, horizon_i, :, :output_dim]) y_pred_peak = scaler.inverse_transform( y_preds[:y_truth.shape[0], horizon_i, :, :output_dim]) y_pred_peak = y_pred_peak[peak] mae_peak = metrics.masked_mae_np(y_pred_peak, y_truth_peak, null_val=0, mode='dcrnn') mape_peak = metrics.masked_mape_np(y_pred_peak, y_truth_peak, null_val=0) rmse_peak = metrics.masked_rmse_np(y_pred_peak, y_truth_peak, null_val=0) mae_peak_list.append(mae_peak) mape_peak_list.append(mape_peak) rmse_peak_list.append(rmse_peak) msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}" if detail: logger.info(msg.format(horizon_i + 1, mae, mape, rmse)) if detail: logger.info('Evaluation_{}_End:'.format(dataset_type)) if format_result: print('-' * 40 + ' 1/4 ' + '-' * 40) for i in range(len(mape_list)): print('{:.2f}'.format(mae_list[i])) print('{:.2f}%'.format(mape_list[i] * 100)) print('{:.2f}'.format(rmse_list[i])) print() print('-' * 40 + ' Peak ' + '-' * 40) for i in range(len(mape_list)): print('{:.2f}'.format(mae_peak_list[i])) print('{:.2f}%'.format(mape_peak_list[i] * 100)) print('{:.2f}'.format(rmse_peak_list[i])) print() else: return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
def evaluate(self, sess, **kwargs): global_step = sess.run(tf.train.get_or_create_global_step()) test_results = self.run_epoch_generator( sess, self._test_model, self._data['test_loader'].get_iterator(), return_output=True, training=False, return_ground_truth=True, task='test') test_loss = test_results['loss'] # utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], # global_step=global_step) utils.add_summary_dict(self._writer, { 'loss/test_loss': test_loss, }, global_step=global_step) # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) y_preds = np.concatenate(test_results['outputs'], axis=0) scaler = self._data['scaler'] y_truths = np.concatenate(test_results['ground_truths'], axis=0) assert y_preds.shape[:3] == y_truths.shape[:3], \ 'NOT {} == {}'.format(y_preds.shape[:3], y_truths.shape[:3]) y_preds_original = [] y_truths_original = [] min_output_value = self._data_kwargs.get('min_output_value') max_output_value = self._data_kwargs.get('max_output_value') clip = (min_output_value is not None) or (max_output_value is not None) if clip: self._kwargs.logger.info('The output values are clipped to range: [{}, {}]'.\ format(min_output_value, max_output_value)) for horizon_i in range(y_truths.shape[1]): y_pred_original = scaler.inverse_transform(y_preds[:, horizon_i, :, 0]) if clip: y_pred_original = \ np.clip(y_pred_original, min_output_value, max_output_value, out=y_pred_original) y_preds_original.append(y_pred_original) self._kwargs.logger.info( stat_str(y_pred_original, horizon_i, 'pred')) y_truth_original = scaler.inverse_transform(y_truths[:, horizon_i, :, 0]) y_truths_original.append(y_truth_original) if not np.all(np.isnan(y_truth_original)): self._kwargs.logger.info( stat_str(y_truth_original, horizon_i, 'true')) for null_val in [0, np.nan]: desc = r'0 excl.' if null_val == 0 else 'any ' rmse = metrics.masked_rmse_np(y_pred_original, y_truth_original, null_val=null_val) r2 = metrics.masked_r2_np(y_pred_original, y_truth_original, null_val=null_val) mae = metrics.masked_mae_np(y_pred_original, y_truth_original, null_val=null_val) mape = metrics.masked_mape_np(y_pred_original, y_truth_original, null_val=null_val) \ if null_val == 0 else np.nan self._kwargs.logger.info( "T+{:02d}|{}|RMSE: {:8.5f}|R2: {:8.4f}|MAE: {:5.2f}|MAPE: {:5.3f}|".\ format(horizon_i + 1, desc, rmse, r2, mae, mape,) ) # utils.add_simple_summary(self._writer, # ['%s_%d' % (item, horizon_i + 1) for item in # ['metric/rmse', 'metric/mape', 'metric/mae']], # [rmse, mape, mae], # global_step=global_step) utils.add_summary_dict( self._writer, { 'metric/rmse_horizon{}'.format(horizon_i + 1): rmse, 'metric/mape_horizon{}'.format(horizon_i + 1): mape, 'metric/mae_horizon{}'.format(horizon_i + 1): mae, 'metric/r2_horizon{}'.format(horizon_i + 1): r2, }, global_step=global_step) outputs = { 'predictions': y_preds_original, 'groundtruth': y_truths_original } return outputs
def _train(self, sess, base_lr, epoch, steps, patience=50, epochs=100, min_learning_rate=2e-6, lr_decay_ratio=0.1, save_model=1, test_every_n_epochs=10, **train_kwargs): history = [] min_val_loss = float('inf') wait = 0 max_to_keep = train_kwargs.get('max_to_keep', 100) saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep) model_filename = train_kwargs.get('model_filename') output_file = train_kwargs.get('preds_file') gt_file = train_kwargs.get('groundtruth_file') if model_filename is not None: saver.restore(sess, model_filename) self._epoch = epoch + 1 else: sess.run(tf.global_variables_initializer()) self._logger.info('Start training ...') while self._epoch <= epochs: # Learning rate schedule. new_lr = max(min_learning_rate, base_lr * (lr_decay_ratio ** np.sum(self._epoch >= np.array(steps)))) self.set_lr(sess=sess, lr=new_lr) start_time = time.time() train_results = self.run_epoch_generator(sess, self._train_model, self._data['train_loader'].get_iterator(), training=True, writer=self._writer) train_loss, train_mae, train_reg = train_results['loss'], train_results['mae'], train_results['reg'] #print ('reg loss is:', train_reg) if train_loss > 1e5: self._logger.warning('Gradient explosion detected. Ending...') break global_step = sess.run(tf.train.get_or_create_global_step()) # Compute validation error. val_results = self.run_epoch_generator(sess, self._test_model, self._data['val_loader'].get_iterator(), training=False) val_loss, val_mae = np.asscalar(val_results['loss']), np.asscalar(val_results['mae']) utils.add_simple_summary(self._writer, ['loss/train_loss', 'metric/train_mae', 'loss/val_loss', 'metric/val_mae'], [train_loss, train_mae, val_loss, val_mae], global_step=global_step) end_time = time.time() message = 'Epoch [{}/{}] ({}) train_mae: {:.4f}, val_mae: {:.4f} lr:{:.6f} {:.1f}s'.format( self._epoch, epochs, global_step, train_mae, val_mae, new_lr, (end_time - start_time)) self._logger.info(message) outputs = self.evaluate(sess) #print (outputs['groundtruth'].shape) y = outputs['groundtruth'][:, :, :, 0] best_pred = outputs['predictions'][:y.shape[0], :, :, 0] y = y.reshape(y.shape[0], -1, order = 'F') # record the corresponding steps for each node first, then node by node best_pred = best_pred.reshape(y.shape[0], -1, order = 'F') scaler = self._data['scaler'] if val_loss <= min_val_loss: wait = 0 if save_model > 0: model_filename = self.save(sess, val_loss) self._logger.info( 'Val loss decrease from %.4f to %.4f, saving to %s' % (min_val_loss, val_loss, model_filename)) min_val_loss = val_loss y = scaler.inverse_transform(y) best_pred = scaler.inverse_transform(best_pred) mae = metrics.masked_mae_np(best_pred, y, null_val=0) self._logger.info( 'Overall Test MAE %.4f' % (mae)) print (best_pred.shape) np.savetxt(output_file, best_pred, delimiter = ',') np.savetxt(gt_file, y, delimiter = ',') else: wait += 1 if wait > patience: self._logger.warning('Early stopping at epoch: %d' % self._epoch) break history.append(val_mae) # Increases epoch. self._epoch += 1 sys.stdout.flush() return np.min(history)
def main(config): logger = config.get_logger('test') graph_pkl_filename = 'data/sensor_graph/adj_mx_unix.pkl' _, _, adj_mat = utils.load_graph_data(graph_pkl_filename) data = utils.load_dataset(dataset_dir='data/METR-LA', batch_size=config["arch"]["args"]["batch_size"], test_batch_size=config["arch"]["args"]["batch_size"]) test_data_loader = data['test_loader'] scaler = data['scaler'] num_test_iteration= math.ceil(data['x_test'].shape[0] / config["arch"]["args"]["batch_size"]) # build model architecture adj_arg = {"adj_mat": adj_mat} model = config.initialize('arch', module_arch, **adj_arg) logger.info(model) logger.info('Loading checkpoint: {} ...'.format(config.resume)) checkpoint = torch.load(config.resume) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) # prepare model for testing device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = model.to(device) model.eval() total_loss = 0.0 y_preds = torch.FloatTensor([]) y_truths = data['y_test'] # (6850, 12, 207, 2) y_truths = scaler.inverse_transform(y_truths) predictions = [] groundtruth = list() with torch.no_grad(): for i, (x, y) in tqdm(enumerate(test_data_loader.get_iterator()), total=num_test_iteration): x = torch.FloatTensor(x).cuda() y = torch.FloatTensor(y).cuda() outputs = model(x, y, 0) # (seq_length+1, batch_size, num_nodes*output_dim) (13, 50, 207*1) y_preds = torch.cat([y_preds, outputs], dim=1) y_preds = torch.transpose(y_preds, 0, 1) y_preds = y_preds.detach().numpy() # cast to numpy array print("--------test results--------") for horizon_i in range(y_truths.shape[1]): y_truth = np.squeeze(y_truths[:, horizon_i, :, 0]) # (6850, 207) y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :]) # (6850, 207) predictions.append(y_pred) groundtruth.append(y_truth) mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0) mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0) rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0) print( "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format( horizon_i + 1, mae, mape, rmse ) ) log = {"Horizon": horizon_i+1, "MAE": mae, "MAPE": mape, "RMSE": rmse} logger.info(log) outputs = { 'predictions': predictions, 'groundtruth': groundtruth } # serialize test data np.savez_compressed('saved/results/dcrnn_predictions.npz', **outputs) print('Predictions saved as {}.'.format('saved/results/dcrnn_predictions.npz'))