def __shared_step_op(self, batch, batch_idx, phase, log=True): img, mask, edge_mask = batch output = self.forward(img) loss_matrix = self.criterion(output, mask) loss = (loss_matrix * (self.edge_weight ** edge_mask)).mean() output_labels = torch.argmax(output, dim=1).view(-1) ground_truths = mask.view(-1) f1 = f1_score(output_labels, ground_truths, num_classes=self.n_classes, class_reduction=self.class_reduction) precision, recall = precision_recall(output_labels, ground_truths, num_classes=self.n_classes, class_reduction=self.class_reduction) if self.n_classes == 2: # use the positive class only for binary case f1 = f1[-1] precision = precision[-1] recall = recall[-1] if log: self.log(f"{phase}/loss", loss, prog_bar=True) self.log(f"{phase}/f1_score", f1, prog_bar=True) self.log(f"{phase}/precision", precision, prog_bar=False) self.log(f"{phase}/recall", recall, prog_bar=False) return {f"{phase}_loss": loss, f"{phase}_f1_score": f1}
def validation_step(self, batch, batch_idx): self.model.eval() tokens, intent_idx, entity_idx, text = batch intent_pred, entity_pred = self.forward(tokens) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] entity_acc = get_token_accuracy( entity_pred.argmax(2), entity_idx, ignore_index=self.dataset.pad_token_id)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) intent_loss = self.intent_loss_fn(intent_pred, intent_idx.squeeze(1)) entity_loss = self.entity_loss_fn( entity_pred.transpose(1, 2), entity_idx.long(), ) return { "val_intent_acc": torch.Tensor([intent_acc]), "val_entity_acc": torch.Tensor([entity_acc]), "val_intent_f1": intent_f1, "val_intent_loss": intent_loss, "val_entity_loss": entity_loss, "val_loss": intent_loss + entity_loss, }
def metrics(self, pred, target, num_classes=3, remove_bg=False, is_swnet=False): if is_swnet: pred[pred == 2] = 1 target[target == 2] = 1 num_classes = 2 confusion_m = mf.confusion_matrix(pred, target, num_classes=num_classes) # acc accuracy = confusion_m.diag().sum() / len(pred) # kappa p0 = accuracy pc = 0 for i in range(confusion_m.shape[0]): pc = pc + confusion_m[i].sum() * confusion_m[:, i].sum() pc = pc / len(pred)**2 kc = (p0 - pc) / (1 - pc) # iou if remove_bg: iou = mf.iou(pred, target, num_classes=num_classes, ignore_index=0) else: iou = mf.iou(pred, target, num_classes=num_classes) f1 = mf.f1_score(pred, target, num_classes=num_classes, class_reduction='none') precision = mf.precision(pred, target, num_classes=num_classes, class_reduction='none') recall = mf.recall(pred, target, num_classes=num_classes, class_reduction='none') return accuracy, kc, iou, f1, precision, recall
def validation_step(self, batch, batch_idx): self.model.eval() tokens, intent_idx, entity_idx = batch feature, intent_pred, entity_pred, entity_loss = self.forward( tokens, entity_idx) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) if entity_idx.sum().item() == 0 or torch.tensor( entity_pred).sum().item() == 0: entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] else: entity_acc = get_token_accuracy( entity_idx.cpu(), torch.tensor(entity_pred).cpu(), ignore_index=self.dataset.pad_token_id)[0] intent_loss = self.intent_loss_fn( intent_pred, intent_idx.long(), ) * self.intent_loss_weight #intent_center_loss = self.intent_center_loss_fn(feature, intent_idx.long(),) * self.intent_loss_weight entity_loss = -entity_loss * self.entity_loss_weight return { "val_intent_acc": torch.Tensor([intent_acc]), "val_intent_f1": torch.Tensor([intent_f1]), "val_entity_acc": torch.Tensor([entity_acc]), "val_loss": intent_loss + entity_loss # + intent_center_loss, }
def validation_epoch_end(self, outputs): val_loss = sum([x["val_loss"] for x in outputs]) pred = torch.cat([x["pred"] for x in outputs]) true = torch.cat([x["true"] for x in outputs]) f_score = metrics.f1_score(pred, true) accuracy = metrics.accuracy(pred, true) out = {"val_loss": val_loss, "val_f_score": f_score, "val_accuracy": accuracy} return {**out, "log": out}
def training_step(self, batch, batch_idx, optimizer_idx): self.model.train() tokens, intent_idx, entity_idx = batch feature, intent_pred, entity_pred, entity_loss = self.forward( tokens, entity_idx) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) if entity_idx.sum().item() == 0 or torch.tensor( entity_pred).sum().item() == 0: entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] else: entity_acc = get_token_accuracy( entity_idx.cpu(), torch.tensor(entity_pred).cpu(), ignore_index=self.dataset.pad_token_id)[0] tensorboard_logs = { "train/intent/acc": intent_acc, "train/intent/f1": intent_f1, "train/entity/acc": entity_acc, } if optimizer_idx == 0: intent_loss = self.intent_loss_fn( intent_pred, intent_idx.long(), ) * self.intent_loss_weight tensorboard_logs["train/intent/loss"] = intent_loss return { "loss": intent_loss, "log": tensorboard_logs, } if optimizer_idx == 1: entity_loss = -entity_loss * self.entity_loss_weight tensorboard_logs["train/entity/loss"] = entity_loss return { "loss": entity_loss, "log": tensorboard_logs, } if optimizer_idx == 2: intent_center_loss = self.intent_center_loss_fn( feature, intent_idx.long(), ) * self.intent_center_loss_weight tensorboard_logs["train/intent/center_loss"] = intent_center_loss return { "loss": intent_center_loss, "log": tensorboard_logs, }
def validation_epoch_end(self, outputs): logits = torch.cat([o["logits"] for o in outputs], dim=0) labels = torch.cat([o["labels"] for o in outputs], dim=0) if self.hparams["multilabel"]: self.log( "val_auroc", np.mean( [ auroc( torch.sigmoid(logits[:, i]), labels[:, i], pos_label=1, ) .detach() .cpu() .item() for i in range(logits.shape[1]) ] ), prog_bar=True, ) else: self.log( "val_f1", f1_score( torch.argmax(logits, dim=1), labels, num_classes=self.hparams["output_size"], class_reduction="macro", ) .detach() .cpu() .item(), prog_bar=True, ) self.log( "val_acc", accuracy(torch.argmax(logits, dim=1), labels) .detach() .cpu() .item(), prog_bar=True, ) self.log( "val_loss", self.loss( logits, labels.type( logits.dtype if self.hparams["multilabel"] else labels.dtype ), ) .detach() .cpu() .item(), prog_bar=True, )
def micro_F1_pytorch(y_true, y_scores): """ Compute Accuracy Args: y_true (np.array): one hot encoded labels y_scores (np.array): model prediction Return: auprc_macro (float): the accuracy """ f1 = plm.f1_score(torch.round(torch.flatten(y_scores)), torch.flatten(y_true), 2) return f1
def test_epoch_end(self, outputs): test_loss = sum([x["test_loss"] for x in outputs]) pred = torch.cat([x["pred"] for x in outputs]) true = torch.cat([x["true"] for x in outputs]) f_score = metrics.f1_score(pred, true) accuracy = metrics.accuracy(pred, true) out = { "test_loss": test_loss, "test_f_score": f_score, "test_accuracy": accuracy, } return {**out, "log": out}
def calculate_metrics(self, pred, target, prvs_metrics=None): # output.cpu().detach().numpy() metrics = { 'accuracy': pl_metrics.accuracy(torch.round(pred), torch.round(target)), 'aucroc': pl_metrics.auroc(pred, target), 'f1_score': pl_metrics.f1_score(torch.round(pred), torch.round(target)) } if prvs_metrics is not None: for key in metrics.keys(): metrics[key] += prvs_metrics[key] metrics['counter'] = prvs_metrics['counter'] + 1 else: metrics['counter'] = 1 return metrics
def training_step(self, batch, batch_idx, optimizer_idx): self.model.train() tokens, intent_idx, entity_idx = batch intent_pred, entity_pred, entity_crf_loss = self.forward( tokens, entity_idx) if torch.isnan(tokens).sum().item() > 0: assert ValueError('tokens error') if torch.isnan(intent_idx).sum().item() > 0: assert ValueError('intent_idx error') if torch.isnan(entity_idx).sum().item() > 0: assert ValueError('entity_idx error') if torch.isnan(intent_pred).sum().item() > 0: assert ValueError('intent_pred error') intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] tensorboard_logs = { "train/intent/acc": intent_acc, "train/intent/f1": intent_f1, "train/entity/acc": entity_acc, } if optimizer_idx == 0: intent_loss = self.intent_loss_fn(intent_pred, intent_idx.long()) tensorboard_logs["train/intent/loss"] = intent_loss return { "loss": intent_loss, "log": tensorboard_logs, } if optimizer_idx == 1: tensorboard_logs["train/entity/loss"] = entity_crf_loss return { "loss": entity_crf_loss, "log": tensorboard_logs, }
def validation_end(self, outputs): scores = {} for dataset_outputs in outputs: # Assuming all val steps returned the same keys keys = dataset_outputs[0].keys() y_hat_key = list(filter(lambda x: 'y_hat' in x, keys))[0] label_key = list(filter(lambda x: 'label' in x, keys))[0] dataset_name = y_hat_key.split('-')[0] y_hats = torch.cat( [x[y_hat_key] for x in dataset_outputs if y_hat_key in x]) labels = torch.cat( [x[label_key] for x in dataset_outputs if label_key in x]) scores[f'avg _{dataset_name}_f1'] = plm.f1_score(y_hats, labels) avg_val_loss = torch.stack( [x['val_loss'] for el in outputs for x in el]).mean() scores['avg_val_loss'] = avg_val_loss return {'log': scores}
def validation_epoch_end(self, outputs): val_loss = sum([x["val_loss"] for x in outputs]) pred = torch.cat([x["pred"] for x in outputs]) true = torch.cat([x["true"] for x in outputs]) f_score = metrics.f1_score(pred, true) accuracy = metrics.accuracy(pred, true) # f_score = sk_metrics.f1_score(pred, true, average="macro") # accuracy = sk_metrics.accuracy_score(pred, true) out = { "val_loss": val_loss, "val_f_score": f_score, "val_accuracy": accuracy, "log": { "val_loss": val_loss, "val_f_score": f_score, "val_accuracy": accuracy, }, } return out
def metrics(self, pred, target, num_classes=3, is_swnet=False): if is_swnet: pred[pred == 2] = 1 target[target == 2] = 1 num_classes = 2 confusion_m = mf.confusion_matrix(pred, target, num_classes=num_classes) # acc # try: accuracy = confusion_m.diag().sum() / len(pred) # except: # print("pred:") # print(pred) # print("target:") # print(target) # print("confusion_m:") # print("confusion_m") # print("---") # accuracy = 0 # kappa # try: p0 = accuracy pc = 0 for i in range(confusion_m.shape[0]): pc = pc + confusion_m[i].sum() * confusion_m[:, i].sum() pc = pc / len(pred)**2 kc = (p0 - pc) / (1 - pc) # if pc != 1: # kc = (p0 - pc) / (1 - pc) # else: # kc = torch.tensor(1.0) # kc.to(p0.device) # except: # kc = 0 f1 = mf.f1_score(pred, target, num_classes=num_classes, class_reduction='none') precision = mf.precision(pred, target, num_classes=num_classes, class_reduction='none') recall = mf.recall(pred, target, num_classes=num_classes, class_reduction='none') return accuracy, kc, f1, precision, recall
def training_step(self, batch, batch_idx, optimizer_idx): self.model.train() tokens, intent_idx, entity_idx = batch feature, = self.forward(tokens) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) tensorboard_logs = { "train/intent/acc": intent_acc, "train/intent/f1": intent_f1, "train/entity/acc": entity_acc, } if optimizer_idx == 0: intent_loss = self.intent_loss_fn(intent_pred, intent_idx.long(),) * self.intent_loss_weight tensorboard_logs["train/intent/loss"] = intent_loss return { "loss": intent_loss, "log": tensorboard_logs, } if optimizer_idx == 1: entity_loss = -entity_loss * self.entity_loss_weight tensorboard_logs["train/entity/loss"] = entity_loss return { "loss": entity_loss, "log": tensorboard_logs, } if optimizer_idx == 2: intent_center_loss = self.intent_center_loss_fn(feature, intent_idx.long(),) * self.intent_center_loss_weight tensorboard_logs["train/intent/center_loss"] = intent_center_loss return { "loss": intent_center_loss, "log": tensorboard_logs, }
def __metrics_per_batch(self, batch): # 1. Forward pass: x, y_true = batch logits = self.forward(x) # 2. Compute loss & performance metrics: # class prediction: if binary (num_outputs == 1) then class label is 0 if logit < 0 else it's 1 # if multiclass then simply run argmax to find the index of the most confident class y_hat = torch.argmax(logits, dim=1) if self.n_outputs > 1 else ( logits > 0.0).squeeze(1).long() loss = self.loss( logits, y_true if self.n_outputs > 1 else y_true.view( (-1, 1)).type_as(x)) num_correct = torch.eq(y_hat, y_true.view(-1)).sum() acc = num_correct.float() / self.batch_size prec = plm.precision(y_hat, y_true, num_classes=self.n_classes) rec = plm.recall(y_hat, y_true, num_classes=self.n_classes) f1 = plm.f1_score(y_hat, y_true, num_classes=self.n_classes) conf_matrix = plm.confusion_matrix(y_hat.long(), y_true.long()) acc1, acc5 = self.__accuracy(logits, y_true, topk=(1, 5)) return (y_true, y_hat, logits, loss, num_correct, acc, prec, rec, f1, conf_matrix, acc1, acc5)
def validation_step(self, batch, batch_idx): self.model.eval() tokens, intent_idx, entity_idx = batch intent_pred, entity_pred, entity_crf_loss = self.forward( tokens, entity_idx) intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0] intent_f1 = f1_score(intent_pred.argmax(1), intent_idx) entity_acc = get_token_accuracy(entity_idx.cpu(), torch.tensor(entity_pred).cpu())[0] intent_loss = self.intent_loss_fn( intent_pred, intent_idx.long(), ) return { "val_intent_acc": torch.Tensor([intent_acc]), "val_intent_f1": torch.Tensor([intent_f1]), "val_entity_acc": torch.Tensor([entity_acc]), "val_loss": intent_loss + entity_crf_loss }
train_f1 = 0 for X_batch, y_batch in train_loader: X_batch, y_batch = X_batch.to(device), y_batch.to(device) optimizer.zero_grad() y_pred = model(X_batch) loss = criterion(y_pred, y_batch) train_loss += loss.item() train_acc += accuracy( y_batch, y_pred) # calculate accuracy (on this single batch) train_recall += recall(y_batch, y_pred) train_precision += train_precision(y_batch, y_pred) train_f1 += f1_score(y_batch, y_pred) loss.backward() optimizer.step() history['train_loss'].append(train_loss / len(train_loader)) history['train_acc'].append(train_acc / len(train_loader)) history['train_recall'].append(train_recall / len(train_loader)) history['train_precision'].append(train_precision / len(train_loader)) history['train_f1'].append(train_f1 / len(train_loader)) logger.info(f"EPOCH: {e} (training)") logger.info( f"{'':<10}Loss{'':<5} ----> {train_loss / len(train_loader):.5f}") logger.info( f"{'':<10}Accuracy{'':<1} ----> {train_acc / len(train_loader):.3f}"
def compute_metric( metric: str, predictions: Union[Sequence, torch.Tensor], labels: Union[Sequence, torch.Tensor], num_classes: int, ): """Compute metric given predictions and target labels Args: metric: name of metric predictions: A sequence of predictions (rouge metrics) or a torch Tensor (other metrics) containing predictions labels: A sequence of labels (rouge metrics) or a torch Tensor (other metrics) containing target labels num_classes: number of classes """ if metric == "accuracy": # Calculate the accuracy if not isinstance(predictions, torch.Tensor): predictions = torch.Tensor(predictions) if not isinstance(labels, torch.Tensor): labels = torch.Tensor(labels) score = lightning_metrics.accuracy( pred=predictions, target=labels, num_classes=num_classes, ).item() elif metric == "f1": # Calculate the f1 if not isinstance(predictions, torch.Tensor): predictions = torch.Tensor(predictions) if not isinstance(labels, torch.Tensor): labels = torch.Tensor(labels) score = lightning_metrics.f1_score( pred=predictions, target=labels, num_classes=num_classes, ).item() elif metric in ("Rouge-1", "Rouge-2", "Rouge-L"): # Calculate rouge scores if metric == "Rouge-1": metric_id = "rouge1" elif metric == "Rouge-2": metric_id = "rouge2" else: metric_id = "rougeLsum" scorer = rouge_scorer.RougeScorer([metric_id], use_stemmer=True) # TODO Remove summarizaton-specific 'format_summary' call # TODO Don't call scorer.score separately for each metric score = statistics.mean( scorer.score(format_summary(reference), format_summary(pred)) [metric].fmeasure for reference, pred in zip(labels, predictions)) elif metric == "class_dist": # Calculate class distribution if not isinstance(labels, torch.Tensor): labels = torch.Tensor(labels) score = (lightning_metrics.to_onehot( tensor=labels, num_classes=num_classes).double().mean(dim=0).tolist()) elif metric == "pred_dist": # Calculate predicted class distribution if not isinstance(predictions, torch.Tensor): predictions = torch.Tensor(predictions) score = (lightning_metrics.to_onehot( tensor=predictions, num_classes=num_classes).double().mean(dim=0).tolist()) else: raise NotImplementedError return score