def collect(self, batch): """ 收集一 batch 的评估输入 Args: batch(dict): 输入数据,字典类型,包含两个Key:(y_true, y_pred): batch['y_true']: (batch_size, 1) batch['y_pred']: (batch_size, 1) """ if not isinstance(batch, dict): raise TypeError('evaluator.collect input is not a dict of user') y_true = batch['y_true'] # tensor y_pred = batch['y_pred'] # tensor if y_true.shape != y_pred.shape: raise ValueError( "batch['y_true'].shape is not equal to batch['y_pred'].shape") for metric in self.metrics: if metric not in self.intermediate_result: self.intermediate_result[metric] = [] for metric in self.metrics: if metric == 'masked_MAE': self.intermediate_result[metric].append( loss.masked_mae_torch(y_pred, y_true, 0).item()) elif metric == 'masked_MSE': self.intermediate_result[metric].append( loss.masked_mse_torch(y_pred, y_true, 0).item()) elif metric == 'masked_RMSE': self.intermediate_result[metric].append( loss.masked_rmse_torch(y_pred, y_true, 0).item()) elif metric == 'masked_MAPE': self.intermediate_result[metric].append( loss.masked_mape_torch(y_pred, y_true, 0).item()) elif metric == 'MAE': self.intermediate_result[metric].append( loss.masked_mae_torch(y_pred, y_true).item()) elif metric == 'MSE': self.intermediate_result[metric].append( loss.masked_mse_torch(y_pred, y_true).item()) elif metric == 'RMSE': self.intermediate_result[metric].append( loss.masked_rmse_torch(y_pred, y_true).item()) elif metric == 'MAPE': self.intermediate_result[metric].append( loss.masked_mape_torch(y_pred, y_true).item()) elif metric == 'R2': self.intermediate_result[metric].append( loss.r2_score_torch(y_pred, y_true).item()) elif metric == 'EVAR': self.intermediate_result[metric].append( loss.explained_variance_score_torch(y_pred, y_true).item())
def eval_on_batch(self, pred, label, mean, std): label = label label = label * std + mean pred = pred * std + mean return loss.masked_mape_torch(pred, label)
def evaluate(self, test_dataloader): """ use model to test data """ self.evaluator.evaluate() node_features = torch.FloatTensor(test_dataloader['node_features']).to( self.device) node_labels = node_features.clone() test_mask = test_dataloader['mask'] self._logger.info('Start evaluating ...') with torch.no_grad(): self.model.eval() output = self.model.predict({'node_features': node_features}) output = self._scaler.inverse_transform(output) node_labels = self._scaler.inverse_transform(node_labels) rmse = loss.masked_rmse_torch(output[test_mask], node_labels[test_mask]) mae = loss.masked_mae_torch(output[test_mask], node_labels[test_mask]) mape = loss.masked_mape_torch(output[test_mask], node_labels[test_mask]) self._logger.info('mae={}, map={}, rmse={}'.format( mae.item(), mape.item(), rmse.item())) return mae.item(), mape.item(), rmse.item()
def eval_on_batch(self, pred, lens, label, mean, std): label = nn.utils.rnn.pack_padded_sequence(label, lens, batch_first=True)[0] label = label label = label * std + mean pred = pred * std + mean return loss.masked_mape_torch(pred, label, eps=self.eps)
def calculate_loss(self, batch): if self.training: T_f_hat = self.predict(batch).unsqueeze(dim=2) T_f = torch.unsqueeze(batch["current_tim"][:, 1:], dim=2) M_f = torch.unsqueeze(batch["masked_current_tim"][:, 1:], dim=1) loss_f = torch.bmm(M_f, torch.pow( (T_f_hat - T_f) / T_f, 2)) / torch.bmm(M_f, M_f.permute( 0, 2, 1)) loss_f = torch.pow(loss_f, 1 / 2) return loss_f.mean() else: pred = self.predict(batch) label = batch["time"] return loss.masked_mape_torch(pred, label)
def collect(self, batch): """ 收集一 batch 的评估输入 Args: batch(dict): 输入数据,字典类型,包含两个Key:(y_true, y_pred): batch['y_true']: (num_samples/batch_size, timeslots, ..., feature_dim) batch['y_pred']: (num_samples/batch_size, timeslots, ..., feature_dim) """ if not isinstance(batch, dict): raise TypeError('evaluator.collect input is not a dict of user') y_true = batch['y_true'] # tensor y_pred = batch['y_pred'] # tensor if y_true.shape != y_pred.shape: raise ValueError( "batch['y_true'].shape is not equal to batch['y_pred'].shape") self.len_timeslots = y_true.shape[1] for i in range(1, self.len_timeslots + 1): for metric in self.metrics: if metric + '@' + str(i) not in self.intermediate_result: self.intermediate_result[metric + '@' + str(i)] = [] if self.mode.lower() == 'average': # 前i个时间步的平均loss for i in range(1, self.len_timeslots + 1): for metric in self.metrics: if metric == 'masked_MAE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mae_torch(y_pred[:, :i], y_true[:, :i], 0).item()) elif metric == 'masked_MSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mse_torch(y_pred[:, :i], y_true[:, :i], 0).item()) elif metric == 'masked_RMSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_rmse_torch(y_pred[:, :i], y_true[:, :i], 0).item()) elif metric == 'masked_MAPE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mape_torch(y_pred[:, :i], y_true[:, :i], 0).item()) elif metric == 'MAE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mae_torch(y_pred[:, :i], y_true[:, :i]).item()) elif metric == 'MSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mse_torch(y_pred[:, :i], y_true[:, :i]).item()) elif metric == 'RMSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_rmse_torch(y_pred[:, :i], y_true[:, :i]).item()) elif metric == 'MAPE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mape_torch(y_pred[:, :i], y_true[:, :i]).item()) elif metric == 'R2': self.intermediate_result[metric + '@' + str(i)].append( loss.r2_score_torch(y_pred[:, :i], y_true[:, :i]).item()) elif metric == 'EVAR': self.intermediate_result[metric + '@' + str(i)].append( loss.explained_variance_score_torch( y_pred[:, :i], y_true[:, :i]).item()) elif self.mode.lower() == 'single': # 第i个时间步的loss for i in range(1, self.len_timeslots + 1): for metric in self.metrics: if metric == 'masked_MAE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mae_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item()) elif metric == 'masked_MSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mse_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item()) elif metric == 'masked_RMSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_rmse_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item()) elif metric == 'masked_MAPE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mape_torch(y_pred[:, i - 1], y_true[:, i - 1], 0).item()) elif metric == 'MAE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mae_torch(y_pred[:, i - 1], y_true[:, i - 1]).item()) elif metric == 'MSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mse_torch(y_pred[:, i - 1], y_true[:, i - 1]).item()) elif metric == 'RMSE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_rmse_torch(y_pred[:, i - 1], y_true[:, i - 1]).item()) elif metric == 'MAPE': self.intermediate_result[metric + '@' + str(i)].append( loss.masked_mape_torch(y_pred[:, i - 1], y_true[:, i - 1]).item()) elif metric == 'R2': self.intermediate_result[metric + '@' + str(i)].append( loss.r2_score_torch(y_pred[:, i - 1], y_true[:, i - 1]).item()) elif metric == 'EVAR': self.intermediate_result[metric + '@' + str(i)].append( loss.explained_variance_score_torch( y_pred[:, i - 1], y_true[:, i - 1]).item()) else: raise ValueError( 'Error parameter evaluator_mode={}, please set `single` or `average`.' .format(self.mode))