def _set_measured_flow(self, rnn_input, pred_forward, m_indicator):
        """

        :param rnn_input: shape(#seq_len, #nflows)
        :param pred_forward: shape(#seq_len, #nflows)
        :param m_indicator: shape(#seq_len, #nflows)
        :return:
        """

        n_flows = rnn_input.shape[0]

        fw_losses = []
        for flow_id in range(m_indicator.shape[1]):
            idx_fw = m_indicator[1:, flow_id]

            # fw_losses.append(error_ratio(y_true=rnn_input[1:, flow_id][idx_fw == 1.0],
            #                              y_pred=pred_forward[:-1, flow_id][idx_fw == 1.0],
            #                              measured_matrix=np.zeros(idx_fw[idx_fw == 1.0].shape)))
            fw_losses.append(metrics.masked_mae_np(preds=pred_forward[:-1, flow_id][idx_fw == 1.0],
                                                   labels=rnn_input[1:, flow_id][idx_fw == 1.0]))

        fw_losses = np.array(fw_losses)
        fw_losses[fw_losses == 0.] = np.max(fw_losses)

        w = self._calculate_flows_weights(fw_losses=fw_losses,
                                          m_indicator=m_indicator)

        sampling = np.zeros(shape=n_flows)
        m = int(self._mon_ratio * n_flows)

        w = w.flatten()
        sorted_idx_w = np.argsort(w)
        sampling[sorted_idx_w[:m]] = 1

        return sampling
Beispiel #2
0
def eval_var(traffic_reading_df, n_lags=3):
    n_forwards = [1, 2]
    y_predicts, y_test = var_predict(traffic_reading_df,
                                     n_forwards=n_forwards,
                                     n_lags=n_lags,
                                     test_ratio=0.2)
    #print("y_predicts type:" + types(y_predicts))
    print("type===========")
    print(type(y_predicts[1].as_matrix()))

    #df_list = pd.DataFrame(y_predicts)
    #df_list.to_csv("./hello.csv", sep=',', index=False)

    logger.info('VAR (lag=%d)' % n_lags)
    logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE')
    for i, horizon in enumerate(n_forwards):
        rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(),
                              labels=y_test.as_matrix(),
                              null_val=0)
        mape = masked_mape_np(preds=y_predicts[i].as_matrix(),
                              labels=y_test.as_matrix(),
                              null_val=0)
        mae = masked_mae_np(preds=y_predicts[i].as_matrix(),
                            labels=y_test.as_matrix(),
                            null_val=0)
        line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
        logger.info(line)
Beispiel #3
0
def evaluate(model,
             dataset,
             dataset_type,
             edge_index,
             edge_attr,
             device,
             output_dim,
             logger,
             detail=True,
             cfg=None,
             format_result=False):
    if detail:
        logger.info('Evaluation_{}_Begin:'.format(dataset_type))
    scaler = dataset['scaler']
    y_preds = run_model(
        model,
        data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(),
        edge_index=edge_index,
        edge_attr=edge_attr,
        device=device,
        output_dim=output_dim)

    y_preds = np.concatenate(y_preds, axis=0)  # concat in batch_size dim.
    mae_list = []
    mape_list = []
    rmse_list = []
    mae_sum = 0
    mape_sum = 0
    rmse_sum = 0
    # horizon = dataset['y_{}'.format(dataset_type)].shape[1]
    horizon = cfg['model']['horizon']
    for horizon_i in range(horizon):
        y_truth = scaler.inverse_transform(
            dataset['y_{}'.format(dataset_type)][:, horizon_i, :, :output_dim])

        y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0],
                                                  horizon_i, :, :output_dim])
        mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn')
        mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
        rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
        mae_sum += mae
        mape_sum += mape
        rmse_sum += rmse
        mae_list.append(mae)
        mape_list.append(mape)
        rmse_list.append(rmse)
        msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}"
        if detail:
            logger.info(msg.format(horizon_i + 1, mae, mape, rmse))
    if detail:
        logger.info('Evaluation_{}_End:'.format(dataset_type))
    if format_result:
        for i in range(len(mape_list)):
            print('{:.2f}'.format(mae_list[i]))
            print('{:.2f}%'.format(mape_list[i] * 100))
            print('{:.2f}'.format(rmse_list[i]))
            print()
    else:
        return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
def eval_historical_average(traffic_reading_df, period):
    y_predict, y_test = historical_average_predict(traffic_reading_df, period=period, test_ratio=0.2)
    rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
    mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
    mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
    logger.info('Historical Average')
    logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
    for horizon in [1, 3, 6, 12]:
        line = 'HA\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
        logger.info(line)
def eval_static(traffic_reading_df):
    logger.info('Static')
    horizons = [1, 3, 6, 12]
    logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE']))
    for horizon in horizons:
        y_predict, y_test = static_predict(traffic_reading_df, n_forward=horizon, test_ratio=0.2)
        rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
        mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
        mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0)
        line = 'Static\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
        logger.info(line)
def eval_var(traffic_reading_df, n_lags=3):
    n_forwards = [1, 3, 6, 12]
    y_predicts, y_test = var_predict(traffic_reading_df, n_forwards=n_forwards, n_lags=n_lags,
                                     test_ratio=0.2)
    logger.info('VAR (lag=%d)' % n_lags)
    logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE')
    for i, horizon in enumerate(n_forwards):
        rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
        mape = masked_mape_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
        mae = masked_mae_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0)
        line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
        logger.info(line)
    def evaluate(self):

        scaler = self._data['scaler']

        y_pred_1, y_pred_2 = self.model.predict(self._data['x_eval'])
        y_pred_1 = scaler.inverse_transform(y_pred_1)
        y_truth_1 = scaler.inverse_transform(self._data['y_eval_1'])
        y_truth_2 = scaler.inverse_transform(self._data['y_eval_2'])

        mse = metrics.masked_mse_np(preds=y_pred_1,
                                    labels=y_truth_1,
                                    null_val=0)
        mae = metrics.masked_mae_np(preds=y_pred_1,
                                    labels=y_truth_1,
                                    null_val=0)
        mape = metrics.masked_mape_np(preds=y_pred_1,
                                      labels=y_truth_1,
                                      null_val=0)
        rmse = metrics.masked_rmse_np(preds=y_pred_1,
                                      labels=y_truth_1,
                                      null_val=0)
        self._logger.info(
            " Forward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
            .format(mse, mae, rmse, mape))

        mse_2 = metrics.masked_mse_np(preds=y_pred_2,
                                      labels=y_truth_2,
                                      null_val=0)
        mae_2 = metrics.masked_mae_np(preds=y_pred_2,
                                      labels=y_truth_2,
                                      null_val=0)
        mape_2 = metrics.masked_mape_np(preds=y_pred_2,
                                        labels=y_truth_2,
                                        null_val=0)
        rmse_2 = metrics.masked_rmse_np(preds=y_pred_2,
                                        labels=y_truth_2,
                                        null_val=0)
        self._logger.info(
            "Backward results: MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
            .format(mse_2, mae_2, rmse_2, mape_2))
    def evaluate(self):
        scaler = self._data['scaler']

        y_pred = self.model.predict(self._data['x_eval'])
        y_pred = scaler.inverse_transform(y_pred)
        y_truth = scaler.inverse_transform(self._data['y_eval'])

        mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
        mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
        mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
        rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
        self._logger.info(
            "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
            .format(1, mse, mae, rmse, mape))
    def evaluate(self, sess):
        global_step = sess.run(tf.train.get_or_create_global_step())
        test_results = self.run_epoch_generator(
            sess,
            self.model,
            self._data['eval_loader'].get_iterator(),
            return_output=True,
            training=False)

        # y_preds:  a list of (batch_size, horizon, num_nodes, output_dim)
        test_loss, y_preds = test_results['loss'], test_results['outputs']
        utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss],
                                 global_step=global_step)

        y_preds = np.concatenate(y_preds, axis=0)
        scaler = self._data['scaler']
        predictions = []
        y_truths = []
        for horizon_i in range(self._data['y_eval'].shape[1]):
            y_truth = scaler.inverse_transform(
                self._data['y_eval'][:, horizon_i, :, 0])
            y_truths.append(y_truth)

            y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0])
            predictions.append(y_pred)

            mse = metrics.masked_mse_np(preds=y_pred,
                                        labels=y_truth,
                                        null_val=0)
            mae = metrics.masked_mae_np(preds=y_pred,
                                        labels=y_truth,
                                        null_val=0)
            mape = metrics.masked_mape_np(preds=y_pred,
                                          labels=y_truth,
                                          null_val=0)
            rmse = metrics.masked_rmse_np(preds=y_pred,
                                          labels=y_truth,
                                          null_val=0)
            self._logger.info(
                "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
                .format(horizon_i + 1, mse, mae, rmse, mape))
            utils.add_simple_summary(self._writer, [
                '%s_%d' % (item, horizon_i + 1)
                for item in ['metric/rmse', 'metric/mae', 'metric/mse']
            ], [rmse, mae, mse],
                                     global_step=global_step)
        outputs = {'predictions': predictions, 'groundtruth': y_truths}
        return outputs
Beispiel #10
0
    def _calculate_metrics(self, prediction_results, metrics_summary, scaler, runId, data_norm, n_metrics=4):
        # y_preds:  a list of (batch_size, horizon, num_nodes, output_dim)
        y_preds = prediction_results['y_preds']
        y_preds = np.concatenate(y_preds, axis=0)

        y_truths = prediction_results['y_truths']
        y_truths = np.concatenate(y_truths, axis=0)
        predictions = []

        for horizon_i in range(self._horizon):
            y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :])

            y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :])
            predictions.append(y_pred)

            mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
            mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
            mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
            rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
            self._logger.info(
                "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format(
                    horizon_i + 1, mse, mae, rmse, mape
                )
            )
            metrics_summary[runId, horizon_i * n_metrics + 0] = mse
            metrics_summary[runId, horizon_i * n_metrics + 1] = mae
            metrics_summary[runId, horizon_i * n_metrics + 2] = rmse
            metrics_summary[runId, horizon_i * n_metrics + 3] = mape

        tm_pred = scaler.inverse_transform(prediction_results['tm_pred'])
        g_truth = scaler.inverse_transform(data_norm[self._seq_len:-self._horizon])
        m_indicator = prediction_results['m_indicator']
        er = metrics.error_ratio(y_pred=tm_pred,
                                 y_true=g_truth,
                                 measured_matrix=m_indicator)
        metrics_summary[runId, -1] = er
        self._logger.info('ER: {}'.format(er))
        self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(runId))
        return metrics_summary
    def _test(self):
        scaler = self._data['scaler']
        results_summary = pd.DataFrame(index=range(self._run_times))
        results_summary['No.'] = range(self._run_times)

        n_metrics = 4
        # Metrics: MSE, MAE, RMSE, MAPE, ER
        metrics_summary = np.zeros(shape=(self._run_times,
                                          self._horizon * n_metrics + 1))

        for i in range(self._run_times):
            print('|--- Running time: {}/{}'.format(i, self._run_times))

            outputs = self._run_tm_prediction()

            tm_pred, m_indicator, y_preds = outputs['tm_pred'], outputs[
                'm_indicator'], outputs['y_preds']

            y_preds = np.concatenate(y_preds, axis=0)
            predictions = []
            y_truths = outputs['y_truths']
            y_truths = np.concatenate(y_truths, axis=0)

            for horizon_i in range(self._horizon):
                y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :])

                y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :])
                predictions.append(y_pred)

                mse = metrics.masked_mse_np(preds=y_pred,
                                            labels=y_truth,
                                            null_val=0)
                mae = metrics.masked_mae_np(preds=y_pred,
                                            labels=y_truth,
                                            null_val=0)
                mape = metrics.masked_mape_np(preds=y_pred,
                                              labels=y_truth,
                                              null_val=0)
                rmse = metrics.masked_rmse_np(preds=y_pred,
                                              labels=y_truth,
                                              null_val=0)
                self._logger.info(
                    "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}"
                    .format(horizon_i + 1, mse, mae, rmse, mape))
                metrics_summary[i, horizon_i * n_metrics + 0] = mse
                metrics_summary[i, horizon_i * n_metrics + 1] = mae
                metrics_summary[i, horizon_i * n_metrics + 2] = rmse
                metrics_summary[i, horizon_i * n_metrics + 3] = mape

            tm_pred = scaler.inverse_transform(tm_pred)
            g_truth = scaler.inverse_transform(
                self._data['test_data_norm'][self._seq_len:-self._horizon])
            er = metrics.error_ratio(y_pred=tm_pred,
                                     y_true=g_truth,
                                     measured_matrix=m_indicator)
            metrics_summary[i, -1] = er

            self._save_results(g_truth=g_truth,
                               pred_tm=tm_pred,
                               m_indicator=m_indicator,
                               tag=str(i))

            print('ER: {}'.format(er))

        for horizon_i in range(self._horizon):
            results_summary['mse_{}'.format(
                horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0]
            results_summary['mae_{}'.format(
                horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1]
            results_summary['rmse_{}'.format(
                horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2]
            results_summary['mape_{}'.format(
                horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3]

        results_summary['er'] = metrics_summary[:, -1]
        results_summary.to_csv(self._log_dir + 'results_summary.csv',
                               index=False)
    def _test(self, sess, **kwargs):

        global_step = sess.run(tf.train.get_or_create_global_step())

        results_summary = pd.DataFrame(index=range(self._run_times))
        results_summary['No.'] = range(self._run_times)

        n_metrics = 4
        # Metrics: MSE, MAE, RMSE, MAPE, ER
        metrics_summary = np.zeros(shape=(self._run_times, self._horizon * n_metrics + 1))

        for i in range(self._run_times):
            self._logger.info('|--- Run time: {}'.format(i))
            # y_test = self._prepare_test_set()

            test_results = self._run_tm_prediction(sess, model=self._test_model)

            # y_preds:  a list of (batch_size, horizon, num_nodes, output_dim)
            test_loss, y_preds = test_results['loss'], test_results['y_preds']
            utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step)

            y_preds = test_results['y_preds']
            y_preds = np.concatenate(y_preds, axis=0)

            y_truths = test_results['y_truths']
            y_truths = np.concatenate(y_truths, axis=0)
            scaler = self._data['scaler']
            predictions = []

            for horizon_i in range(self._horizon):
                y_truth = scaler.inverse_transform(y_truths[:, horizon_i, :, 0])

                y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0])
                predictions.append(y_pred)

                mse = metrics.masked_mse_np(preds=y_pred, labels=y_truth, null_val=0)
                mae = metrics.masked_mae_np(preds=y_pred, labels=y_truth, null_val=0)
                mape = metrics.masked_mape_np(preds=y_pred, labels=y_truth, null_val=0)
                rmse = metrics.masked_rmse_np(preds=y_pred, labels=y_truth, null_val=0)
                self._logger.info(
                    "Horizon {:02d}, MSE: {:.2f}, MAE: {:.2f}, RMSE: {:.2f}, MAPE: {:.4f}".format(
                        horizon_i + 1, mse, mae, rmse, mape
                    )
                )
                metrics_summary[i, horizon_i * n_metrics + 0] = mse
                metrics_summary[i, horizon_i * n_metrics + 1] = mae
                metrics_summary[i, horizon_i * n_metrics + 2] = rmse
                metrics_summary[i, horizon_i * n_metrics + 3] = mape

            tm_pred = scaler.inverse_transform(test_results['tm_pred'])
            g_truth = scaler.inverse_transform(self._data['test_data_norm'][self._seq_len:-self._horizon])
            m_indicator = test_results['m_indicator']
            er = error_ratio(y_pred=tm_pred,
                             y_true=g_truth,
                             measured_matrix=m_indicator)
            metrics_summary[i, -1] = er

            self._save_results(g_truth=g_truth, pred_tm=tm_pred, m_indicator=m_indicator, tag=str(i))

            print('ER: {}'.format(er))

        for horizon_i in range(self._horizon):
            results_summary['mse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 0]
            results_summary['mae_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 1]
            results_summary['rmse_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 2]
            results_summary['mape_{}'.format(horizon_i)] = metrics_summary[:, horizon_i * n_metrics + 3]

        results_summary['er'] = metrics_summary[:, -1]
        results_summary.to_csv(self._log_dir + 'results_summary.csv', index=False)

        return
Beispiel #13
0
def evaluate(model,
             dataset,
             dataset_type,
             edge_index,
             edge_attr,
             device,
             output_dim,
             logger,
             detail=True,
             cfg=None,
             format_result=False):
    if detail:
        logger.info('Evaluation_{}_Begin:'.format(dataset_type))
    scaler = dataset['scaler']
    y_preds = run_model(
        model,
        data_iterator=dataset['{}_loader'.format(dataset_type)].get_iterator(),
        edge_index=edge_index,
        edge_attr=edge_attr,
        device=device,
        output_dim=output_dim)

    y_preds = np.concatenate(y_preds, axis=0)  # concat in batch_size dim.
    mae_sum = 0
    mape_sum = 0
    rmse_sum = 0
    mae_list = []
    mape_list = []
    rmse_list = []
    mae_peak_list = []
    mape_peak_list = []
    rmse_peak_list = []
    # horizon = dataset['y_{}'.format(dataset_type)].shape[1]
    horizon = cfg['model']['horizon']
    ytime = dataset['ytime_{}'.format(dataset_type)][:, 0]
    ytime_pd = pd.to_datetime(ytime.flatten())
    morning_peak = np.logical_and(
        (ytime_pd.hour * 60 + ytime_pd.minute) >= 7 * 60 + 30,
        (ytime_pd.hour * 60 + ytime_pd.minute) <= 8 * 60 + 30)
    evening_peak = np.logical_and(
        (ytime_pd.hour * 60 + ytime_pd.minute) >= 17 * 60 + 30,
        (ytime_pd.hour * 60 + ytime_pd.minute) <= 18 * 60 + 30)
    peak = np.logical_or(morning_peak, evening_peak)
    for horizon_i in range(horizon):
        y_truth = scaler.inverse_transform(
            dataset['y_{}'.format(dataset_type)][:, horizon_i,
                                                 shtop72_index, :output_dim])

        y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0], horizon_i,
                                                  shtop72_index, :output_dim])
        mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0, mode='dcrnn')
        mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
        rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
        mae_sum += mae
        mape_sum += mape
        rmse_sum += rmse
        mae_list.append(mae)
        mape_list.append(mape)
        rmse_list.append(rmse)

        # Peak
        y_truth_peak = scaler.inverse_transform(
            dataset['y_{}'.format(dataset_type)][peak,
                                                 horizon_i, :, :output_dim])
        y_pred_peak = scaler.inverse_transform(
            y_preds[:y_truth.shape[0], horizon_i, :, :output_dim])
        y_pred_peak = y_pred_peak[peak]
        mae_peak = metrics.masked_mae_np(y_pred_peak,
                                         y_truth_peak,
                                         null_val=0,
                                         mode='dcrnn')
        mape_peak = metrics.masked_mape_np(y_pred_peak,
                                           y_truth_peak,
                                           null_val=0)
        rmse_peak = metrics.masked_rmse_np(y_pred_peak,
                                           y_truth_peak,
                                           null_val=0)
        mae_peak_list.append(mae_peak)
        mape_peak_list.append(mape_peak)
        rmse_peak_list.append(rmse_peak)

        msg = "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}"
        if detail:
            logger.info(msg.format(horizon_i + 1, mae, mape, rmse))
    if detail:
        logger.info('Evaluation_{}_End:'.format(dataset_type))
    if format_result:
        print('-' * 40 + ' 1/4 ' + '-' * 40)
        for i in range(len(mape_list)):
            print('{:.2f}'.format(mae_list[i]))
            print('{:.2f}%'.format(mape_list[i] * 100))
            print('{:.2f}'.format(rmse_list[i]))
            print()
        print('-' * 40 + ' Peak ' + '-' * 40)
        for i in range(len(mape_list)):
            print('{:.2f}'.format(mae_peak_list[i]))
            print('{:.2f}%'.format(mape_peak_list[i] * 100))
            print('{:.2f}'.format(rmse_peak_list[i]))
            print()
    else:
        return mae_sum / horizon, mape_sum / horizon, rmse_sum / horizon
Beispiel #14
0
    def evaluate(self, sess, **kwargs):
        global_step = sess.run(tf.train.get_or_create_global_step())
        test_results = self.run_epoch_generator(
            sess,
            self._test_model,
            self._data['test_loader'].get_iterator(),
            return_output=True,
            training=False,
            return_ground_truth=True,
            task='test')

        test_loss = test_results['loss']
        # utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss],
        #                          global_step=global_step)
        utils.add_summary_dict(self._writer, {
            'loss/test_loss': test_loss,
        },
                               global_step=global_step)

        # y_preds:  a list of (batch_size, horizon, num_nodes, output_dim)
        y_preds = np.concatenate(test_results['outputs'], axis=0)

        scaler = self._data['scaler']

        y_truths = np.concatenate(test_results['ground_truths'], axis=0)

        assert y_preds.shape[:3] == y_truths.shape[:3], \
            'NOT {} == {}'.format(y_preds.shape[:3], y_truths.shape[:3])

        y_preds_original = []
        y_truths_original = []

        min_output_value = self._data_kwargs.get('min_output_value')
        max_output_value = self._data_kwargs.get('max_output_value')
        clip = (min_output_value is not None) or (max_output_value is not None)
        if clip:
            self._kwargs.logger.info('The output values are clipped to range: [{}, {}]'.\
                              format(min_output_value, max_output_value))

        for horizon_i in range(y_truths.shape[1]):
            y_pred_original = scaler.inverse_transform(y_preds[:, horizon_i, :,
                                                               0])
            if clip:
                y_pred_original = \
                    np.clip(y_pred_original, min_output_value, max_output_value,
                            out=y_pred_original)
            y_preds_original.append(y_pred_original)
            self._kwargs.logger.info(
                stat_str(y_pred_original, horizon_i, 'pred'))

            y_truth_original = scaler.inverse_transform(y_truths[:,
                                                                 horizon_i, :,
                                                                 0])
            y_truths_original.append(y_truth_original)

            if not np.all(np.isnan(y_truth_original)):
                self._kwargs.logger.info(
                    stat_str(y_truth_original, horizon_i, 'true'))

                for null_val in [0, np.nan]:
                    desc = r'0 excl.' if null_val == 0 else 'any    '
                    rmse = metrics.masked_rmse_np(y_pred_original,
                                                  y_truth_original,
                                                  null_val=null_val)
                    r2 = metrics.masked_r2_np(y_pred_original,
                                              y_truth_original,
                                              null_val=null_val)
                    mae = metrics.masked_mae_np(y_pred_original,
                                                y_truth_original,
                                                null_val=null_val)
                    mape = metrics.masked_mape_np(y_pred_original, y_truth_original, null_val=null_val) \
                        if null_val == 0 else np.nan
                    self._kwargs.logger.info(
                        "T+{:02d}|{}|RMSE: {:8.5f}|R2: {:8.4f}|MAE: {:5.2f}|MAPE: {:5.3f}|".\
                            format(horizon_i + 1, desc, rmse, r2, mae, mape,)
                        )

                # utils.add_simple_summary(self._writer,
                #                          ['%s_%d' % (item, horizon_i + 1) for item in
                #                           ['metric/rmse', 'metric/mape', 'metric/mae']],
                #                          [rmse, mape, mae],
                #                          global_step=global_step)
                utils.add_summary_dict(
                    self._writer, {
                        'metric/rmse_horizon{}'.format(horizon_i + 1): rmse,
                        'metric/mape_horizon{}'.format(horizon_i + 1): mape,
                        'metric/mae_horizon{}'.format(horizon_i + 1): mae,
                        'metric/r2_horizon{}'.format(horizon_i + 1): r2,
                    },
                    global_step=global_step)
        outputs = {
            'predictions': y_preds_original,
            'groundtruth': y_truths_original
        }
        return outputs
Beispiel #15
0
    def _train(self, sess, base_lr, epoch, steps, patience=50, epochs=100,
               min_learning_rate=2e-6, lr_decay_ratio=0.1, save_model=1,
               test_every_n_epochs=10, **train_kwargs):
        history = []
        min_val_loss = float('inf')
        wait = 0

        max_to_keep = train_kwargs.get('max_to_keep', 100)
        saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep)
        model_filename = train_kwargs.get('model_filename')

        output_file = train_kwargs.get('preds_file')
        gt_file = train_kwargs.get('groundtruth_file')
        if model_filename is not None:
            saver.restore(sess, model_filename)
            self._epoch = epoch + 1
        else:
            sess.run(tf.global_variables_initializer())
        self._logger.info('Start training ...')

        while self._epoch <= epochs:
            # Learning rate schedule.
            new_lr = max(min_learning_rate, base_lr * (lr_decay_ratio ** np.sum(self._epoch >= np.array(steps))))
            self.set_lr(sess=sess, lr=new_lr)

            start_time = time.time()
            train_results = self.run_epoch_generator(sess, self._train_model,
                                                     self._data['train_loader'].get_iterator(),
                                                     training=True,
                                                     writer=self._writer)
            train_loss, train_mae, train_reg = train_results['loss'], train_results['mae'], train_results['reg']
            #print ('reg loss is:', train_reg)
            if train_loss > 1e5:
                self._logger.warning('Gradient explosion detected. Ending...')
                break

            global_step = sess.run(tf.train.get_or_create_global_step())
            # Compute validation error.
            val_results = self.run_epoch_generator(sess, self._test_model,
                                                   self._data['val_loader'].get_iterator(),
                                                   training=False)
            val_loss, val_mae = np.asscalar(val_results['loss']), np.asscalar(val_results['mae'])

            utils.add_simple_summary(self._writer,
                                     ['loss/train_loss', 'metric/train_mae', 'loss/val_loss', 'metric/val_mae'],
                                     [train_loss, train_mae, val_loss, val_mae], global_step=global_step)
            end_time = time.time()
            message = 'Epoch [{}/{}] ({}) train_mae: {:.4f}, val_mae: {:.4f} lr:{:.6f} {:.1f}s'.format(
                self._epoch, epochs, global_step, train_mae, val_mae, new_lr, (end_time - start_time))
            self._logger.info(message)
            outputs = self.evaluate(sess)
            #print (outputs['groundtruth'].shape)
            y = outputs['groundtruth'][:, :, :, 0] 
            best_pred = outputs['predictions'][:y.shape[0], :, :, 0] 
            y = y.reshape(y.shape[0], -1, order = 'F') # record the corresponding steps for each node first, then node by node
            best_pred = best_pred.reshape(y.shape[0], -1, order = 'F')
            scaler = self._data['scaler'] 

            if val_loss <= min_val_loss:
                wait = 0
                if save_model > 0:
                    model_filename = self.save(sess, val_loss)
                self._logger.info(
                    'Val loss decrease from %.4f to %.4f, saving to %s' % (min_val_loss, val_loss, model_filename))
                min_val_loss = val_loss
                y = scaler.inverse_transform(y)
                best_pred = scaler.inverse_transform(best_pred)
                mae = metrics.masked_mae_np(best_pred, y, null_val=0)

                self._logger.info(
                    'Overall Test MAE %.4f' % (mae))
                print (best_pred.shape)
                np.savetxt(output_file, best_pred, delimiter = ',')
                np.savetxt(gt_file, y, delimiter = ',')
            else:
                wait += 1
                if wait > patience:
                    self._logger.warning('Early stopping at epoch: %d' % self._epoch)
                    break

            history.append(val_mae)
            # Increases epoch.
            self._epoch += 1

            sys.stdout.flush()
        return np.min(history)
Beispiel #16
0
def main(config):
    logger = config.get_logger('test')

    graph_pkl_filename = 'data/sensor_graph/adj_mx_unix.pkl'
    _, _, adj_mat = utils.load_graph_data(graph_pkl_filename)
    data = utils.load_dataset(dataset_dir='data/METR-LA',
                              batch_size=config["arch"]["args"]["batch_size"],
                              test_batch_size=config["arch"]["args"]["batch_size"])
    test_data_loader = data['test_loader']
    scaler = data['scaler']
    num_test_iteration= math.ceil(data['x_test'].shape[0] / config["arch"]["args"]["batch_size"])

    # build model architecture
    adj_arg = {"adj_mat": adj_mat}
    model = config.initialize('arch', module_arch, **adj_arg)
    logger.info(model)

    logger.info('Loading checkpoint: {} ...'.format(config.resume))
    checkpoint = torch.load(config.resume)
    state_dict = checkpoint['state_dict']
    if config['n_gpu'] > 1:
        model = torch.nn.DataParallel(model)
    model.load_state_dict(state_dict)

    # prepare model for testing
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model = model.to(device)
    model.eval()

    total_loss = 0.0

    y_preds = torch.FloatTensor([])
    y_truths = data['y_test']  # (6850, 12, 207, 2)
    y_truths = scaler.inverse_transform(y_truths)
    predictions = []
    groundtruth = list()

    with torch.no_grad():
        for i, (x, y) in tqdm(enumerate(test_data_loader.get_iterator()), total=num_test_iteration):
            x = torch.FloatTensor(x).cuda()
            y = torch.FloatTensor(y).cuda()
            outputs = model(x, y, 0)  # (seq_length+1, batch_size, num_nodes*output_dim)  (13, 50, 207*1)
            y_preds = torch.cat([y_preds, outputs], dim=1)
    y_preds = torch.transpose(y_preds, 0, 1)
    y_preds = y_preds.detach().numpy()  # cast to numpy array
    print("--------test results--------")
    for horizon_i in range(y_truths.shape[1]):
        y_truth = np.squeeze(y_truths[:, horizon_i, :, 0])  # (6850, 207)

        y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :])  # (6850, 207)
        predictions.append(y_pred)
        groundtruth.append(y_truth)

        mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0)
        mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0)
        rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0)
        print(
            "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format(
                horizon_i + 1, mae, mape, rmse
            )
        )
        log = {"Horizon": horizon_i+1, "MAE": mae, "MAPE": mape, "RMSE": rmse}
        logger.info(log)
    outputs = {
        'predictions': predictions,
        'groundtruth': groundtruth
    }

    # serialize test data
    np.savez_compressed('saved/results/dcrnn_predictions.npz', **outputs)
    print('Predictions saved as {}.'.format('saved/results/dcrnn_predictions.npz'))