def test_v1_4_0_deprecated_metrics(): from pytorch_lightning.metrics.functional.classification import stat_scores_multiple_classes with pytest.deprecated_call(match='will be removed in v1.4'): stat_scores_multiple_classes(pred=torch.tensor([0, 1]), target=torch.tensor([0, 1])) from pytorch_lightning.metrics.functional.classification import iou with pytest.deprecated_call(match='will be removed in v1.4'): iou(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) from pytorch_lightning.metrics.functional.classification import recall with pytest.deprecated_call(match='will be removed in v1.4'): recall(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) from pytorch_lightning.metrics.functional.classification import precision with pytest.deprecated_call(match='will be removed in v1.4'): precision(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) from pytorch_lightning.metrics.functional.classification import precision_recall with pytest.deprecated_call(match='will be removed in v1.4'): precision_recall(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) # Testing deprecation of class_reduction arg in the *new* precision from pytorch_lightning.metrics.functional import precision with pytest.deprecated_call(match='will be removed in v1.4'): precision(torch.randint(0, 2, (10, )), torch.randint(0, 2, (10, )), class_reduction='micro') # Testing deprecation of class_reduction arg in the *new* recall from pytorch_lightning.metrics.functional import recall with pytest.deprecated_call(match='will be removed in v1.4'): recall(torch.randint(0, 2, (10, )), torch.randint(0, 2, (10, )), class_reduction='micro') from pytorch_lightning.metrics.functional.classification import auc with pytest.deprecated_call(match='will be removed in v1.4'): auc(torch.rand(10, ).sort().values, torch.rand(10, )) from pytorch_lightning.metrics.functional.classification import auroc with pytest.deprecated_call(match='will be removed in v1.4'): auroc(torch.rand(10, ), torch.randint(0, 2, (10, ))) from pytorch_lightning.metrics.functional.classification import multiclass_auroc with pytest.deprecated_call(match='will be removed in v1.4'): multiclass_auroc(torch.rand(20, 5).softmax(dim=-1), torch.randint(0, 5, (20, )), num_classes=5) from pytorch_lightning.metrics.functional.classification import auc_decorator with pytest.deprecated_call(match='will be removed in v1.4'): auc_decorator() from pytorch_lightning.metrics.functional.classification import multiclass_auc_decorator with pytest.deprecated_call(match='will be removed in v1.4'): multiclass_auc_decorator()
def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: """ Actual metric computation Args: pred: predicted labels target: ground truth labels Return: A Tensor with the classification score. """ return recall(pred=pred, target=target, num_classes=self.num_classes, reduction=self.reduction)
def test_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict: """ Runs one training step. This usually consists in the forward function followed by the loss function. :param batch: The output of your dataloader. :param batch_nb: Integer displaying which batch this is Returns: - dictionary containing the loss and the metrics to be added to the lightning logger. """ inputs, targets = batch model_out = self.forward(**inputs) loss_val = self.loss(model_out, targets) # in DP mode (default) make sure if result is scalar, there's another dim in the beginning if self.trainer.use_dp or self.trainer.use_ddp2: loss_val = loss_val.unsqueeze(0) self.log('test_loss',loss_val) y_hat=model_out['logits'] labels_hat = torch.argmax(y_hat, dim=1) y=targets['labels'] f1 = metrics.f1_score(labels_hat,y, class_reduction='weighted') prec =metrics.precision(labels_hat,y, class_reduction='weighted') recall = metrics.recall(labels_hat,y, class_reduction='weighted') acc = metrics.accuracy(labels_hat,y, class_reduction='weighted') # auroc = metrics.multiclass_auroc(labels_hat, y) self.log('test_batch_prec',prec) self.log('test_batch_f1',f1) self.log('test_batch_recall',recall) self.log('test_batch_weighted_acc', acc) # self.log('test_batch_auc_roc', auroc) from pytorch_lightning.metrics.functional import confusion_matrix # TODO CHANGE THIS # return (labels_hat, y) cm = confusion_matrix(preds = labels_hat,target=y,normalize=None, num_classes=50) # cm = confusion_matrix(preds = labels_hat,target=y,normalize=False, num_classes=len(y.unique())) self.test_conf_matrices.append(cm)
def validation_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict: """ Similar to the training step but with the model in eval mode. Returns: - dictionary passed to the validation_end function. """ inputs, targets = batch model_out = self.forward(**inputs) loss_val = self.loss(model_out, targets) y = targets["labels"] y_hat = model_out["logits"] # acc labels_hat = torch.argmax(y_hat, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) val_acc = torch.tensor(val_acc) if self.on_gpu: val_acc = val_acc.cuda(loss_val.device.index) # in DP mode (default) make sure if result is scalar, there's another dim in the beginning if self.trainer.use_dp or self.trainer.use_ddp2: loss_val = loss_val.unsqueeze(0) val_acc = val_acc.unsqueeze(0) self.log('val_loss',loss_val) f1 = metrics.f1_score(labels_hat, y,class_reduction='weighted') prec =metrics.precision(labels_hat, y,class_reduction='weighted') recall = metrics.recall(labels_hat, y,class_reduction='weighted') acc = metrics.accuracy(labels_hat, y,class_reduction='weighted') # auroc = metrics.multiclass_auroc(y_hat,y) self.log('val_prec',prec) self.log('val_f1',f1) self.log('val_recall',recall) self.log('val_acc_weighted', acc)
def test_precision_recall(pred, target, expected_prec, expected_rec): prec = precision(pred, target, class_reduction='none') rec = recall(pred, target, class_reduction='none') assert torch.allclose(torch.tensor(expected_prec).to(prec), prec) assert torch.allclose(torch.tensor(expected_rec).to(rec), rec)