def test_v1_4_0_deprecated_metrics(): from pytorch_lightning.metrics.functional.classification import stat_scores_multiple_classes with pytest.deprecated_call(match='will be removed in v1.4'): stat_scores_multiple_classes(pred=torch.tensor([0, 1]), target=torch.tensor([0, 1])) from pytorch_lightning.metrics.functional.classification import iou with pytest.deprecated_call(match='will be removed in v1.4'): iou(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) from pytorch_lightning.metrics.functional.classification import recall with pytest.deprecated_call(match='will be removed in v1.4'): recall(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) from pytorch_lightning.metrics.functional.classification import precision with pytest.deprecated_call(match='will be removed in v1.4'): precision(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) from pytorch_lightning.metrics.functional.classification import precision_recall with pytest.deprecated_call(match='will be removed in v1.4'): precision_recall(torch.randint(0, 2, (10, 3, 3)), torch.randint(0, 2, (10, 3, 3))) # Testing deprecation of class_reduction arg in the *new* precision from pytorch_lightning.metrics.functional import precision with pytest.deprecated_call(match='will be removed in v1.4'): precision(torch.randint(0, 2, (10, )), torch.randint(0, 2, (10, )), class_reduction='micro') # Testing deprecation of class_reduction arg in the *new* recall from pytorch_lightning.metrics.functional import recall with pytest.deprecated_call(match='will be removed in v1.4'): recall(torch.randint(0, 2, (10, )), torch.randint(0, 2, (10, )), class_reduction='micro') from pytorch_lightning.metrics.functional.classification import auc with pytest.deprecated_call(match='will be removed in v1.4'): auc(torch.rand(10, ).sort().values, torch.rand(10, )) from pytorch_lightning.metrics.functional.classification import auroc with pytest.deprecated_call(match='will be removed in v1.4'): auroc(torch.rand(10, ), torch.randint(0, 2, (10, ))) from pytorch_lightning.metrics.functional.classification import multiclass_auroc with pytest.deprecated_call(match='will be removed in v1.4'): multiclass_auroc(torch.rand(20, 5).softmax(dim=-1), torch.randint(0, 5, (20, )), num_classes=5) from pytorch_lightning.metrics.functional.classification import auc_decorator with pytest.deprecated_call(match='will be removed in v1.4'): auc_decorator() from pytorch_lightning.metrics.functional.classification import multiclass_auc_decorator with pytest.deprecated_call(match='will be removed in v1.4'): multiclass_auc_decorator()
def test_multiclass_auroc(): with pytest.raises( ValueError, match= r".*probabilities, i.e. they should sum up to 1.0 over classes"): _ = multiclass_auroc(pred=torch.tensor([[0.9, 0.9], [1.0, 0]]), target=torch.tensor([0, 1])) with pytest.raises( ValueError, match= r".*not defined when all of the classes do not occur in the target.*" ): _ = multiclass_auroc(pred=torch.rand((4, 3)).softmax(dim=1), target=torch.tensor([1, 0, 1, 0])) with pytest.raises( ValueError, match= r".*does not equal the number of classes passed in 'num_classes'.*" ): _ = multiclass_auroc(pred=torch.rand((5, 4)).softmax(dim=1), target=torch.tensor([0, 1, 2, 2, 3]), num_classes=6)
def test_multiclass_auroc_against_sklearn(n_cls): device = 'cuda' if torch.cuda.is_available() else 'cpu' n_samples = 300 pred = torch.rand(n_samples, n_cls, device=device).softmax(dim=1) target = torch.randint(n_cls, (n_samples, ), device=device) # Make sure target includes all class labels so that multiclass AUROC is defined target[10:10 + n_cls] = torch.arange(n_cls) pl_score = multiclass_auroc(pred, target) # For the binary case, sklearn expects an (n_samples,) array of probabilities of # the positive class pred = pred[:, 1] if n_cls == 2 else pred sk_score = sk_roc_auc_score(target.cpu().detach().numpy(), pred.cpu().detach().numpy(), multi_class="ovr") sk_score = torch.tensor(sk_score, dtype=torch.float, device=device) assert torch.allclose(sk_score, pl_score)