def test_v1_5_metrics_utils(): x = torch.tensor([1, 2, 3]) with pytest.deprecated_call(match="It will be removed in v1.5.0"): assert torch.equal(to_onehot(x), torch.Tensor([[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]).to(int)) with pytest.deprecated_call(match="It will be removed in v1.5.0"): assert get_num_classes(torch.tensor([1, 2, 3]), torch.tensor([1, 2, 0])) == 4 x = torch.tensor([[1.1, 2.0, 3.0], [2.0, 1.0, 0.5]]) with pytest.deprecated_call(match="It will be removed in v1.5.0"): assert torch.equal(select_topk(x, topk=2), torch.Tensor([[0, 1, 1], [1, 1, 0]]).to(torch.int32)) x = torch.tensor([[0.2, 0.5], [0.9, 0.1]]) with pytest.deprecated_call(match="It will be removed in v1.5.0"): assert torch.equal(to_categorical(x), torch.Tensor([1, 0]).to(int))
def __multiclass_roc( pred: torch.Tensor, target: torch.Tensor, sample_weight: Optional[Sequence] = None, num_classes: Optional[int] = None, ) -> Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: """ Computes the Receiver Operating Characteristic (ROC) for multiclass predictors. .. warning:: Deprecated Args: pred: estimated probabilities target: ground-truth labels sample_weight: sample weights num_classes: number of classes (default: None, computes automatically from data) Return: returns roc for each class. Number of classes, false-positive rate (fpr), true-positive rate (tpr), thresholds Example: >>> pred = torch.tensor([[0.85, 0.05, 0.05, 0.05], ... [0.05, 0.85, 0.05, 0.05], ... [0.05, 0.05, 0.85, 0.05], ... [0.05, 0.05, 0.05, 0.85]]) >>> target = torch.tensor([0, 1, 3, 2]) >>> __multiclass_roc(pred, target) # doctest: +NORMALIZE_WHITESPACE ((tensor([0., 0., 1.]), tensor([0., 1., 1.]), tensor([1.8500, 0.8500, 0.0500])), (tensor([0., 0., 1.]), tensor([0., 1., 1.]), tensor([1.8500, 0.8500, 0.0500])), (tensor([0.0000, 0.3333, 1.0000]), tensor([0., 0., 1.]), tensor([1.8500, 0.8500, 0.0500])), (tensor([0.0000, 0.3333, 1.0000]), tensor([0., 0., 1.]), tensor([1.8500, 0.8500, 0.0500]))) """ num_classes = get_num_classes(pred, target, num_classes) class_roc_vals = [] for c in range(num_classes): pred_c = pred[:, c] class_roc_vals.append( __roc(pred=pred_c, target=target, sample_weight=sample_weight, pos_label=c)) return tuple(class_roc_vals)
def stat_scores_multiple_classes( pred: torch.Tensor, target: torch.Tensor, num_classes: Optional[int] = None, argmax_dim: int = 1, reduction: str = 'none', ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ Calculates the number of true positive, false positive, true negative and false negative for each class .. warning :: Deprecated in favor of :func:`~pytorch_lightning.metrics.functional.stat_scores` Raises: ValueError: If ``reduction`` is not one of ``"none"``, ``"sum"`` or ``"elementwise_mean"``. """ rank_zero_warn( "This `stat_scores_multiple_classes` was deprecated in v1.2.0 in favor of" " `from pytorch_lightning.metrics.functional import stat_scores`." " It will be removed in v1.4.0", DeprecationWarning) if pred.ndim == target.ndim + 1: pred = to_categorical(pred, argmax_dim=argmax_dim) num_classes = get_num_classes(pred=pred, target=target, num_classes=num_classes) if pred.dtype != torch.bool: pred = pred.clamp_max(max=num_classes) if target.dtype != torch.bool: target = target.clamp_max(max=num_classes) possible_reductions = ('none', 'sum', 'elementwise_mean') if reduction not in possible_reductions: raise ValueError("reduction type %s not supported" % reduction) if reduction == 'none': pred = pred.view((-1, )).long() target = target.view((-1, )).long() tps = torch.zeros((num_classes + 1, ), device=pred.device) fps = torch.zeros((num_classes + 1, ), device=pred.device) fns = torch.zeros((num_classes + 1, ), device=pred.device) sups = torch.zeros((num_classes + 1, ), device=pred.device) match_true = (pred == target).float() match_false = 1 - match_true tps.scatter_add_(0, pred, match_true) fps.scatter_add_(0, pred, match_false) fns.scatter_add_(0, target, match_false) tns = pred.size(0) - (tps + fps + fns) sups.scatter_add_(0, target, torch.ones_like(match_true)) tps = tps[:num_classes] fps = fps[:num_classes] tns = tns[:num_classes] fns = fns[:num_classes] sups = sups[:num_classes] elif reduction == 'sum' or reduction == 'elementwise_mean': count_match_true = (pred == target).sum().float() oob_tp, oob_fp, oob_tn, oob_fn, oob_sup = stat_scores( pred, target, num_classes, argmax_dim) tps = count_match_true - oob_tp fps = pred.nelement() - count_match_true - oob_fp fns = pred.nelement() - count_match_true - oob_fn tns = pred.nelement() * (num_classes + 1) - (tps + fps + fns + oob_tn) sups = pred.nelement() - oob_sup.float() if reduction == 'elementwise_mean': tps /= num_classes fps /= num_classes fns /= num_classes tns /= num_classes sups /= num_classes return tps.float(), fps.float(), tns.float(), fns.float(), sups.float()
def iou( pred: torch.Tensor, target: torch.Tensor, ignore_index: Optional[int] = None, absent_score: float = 0.0, threshold: float = 0.5, num_classes: Optional[int] = None, reduction: str = 'elementwise_mean', ) -> torch.Tensor: r""" Computes `Intersection over union, or Jaccard index calculation <https://en.wikipedia.org/wiki/Jaccard_index>`_: .. math:: J(A,B) = \frac{|A\cap B|}{|A\cup B|} Where: :math:`A` and :math:`B` are both tensors of the same size, containing integer class values. They may be subject to conversion from input data (see description below). Note that it is different from box IoU. If preds and target are the same shape and preds is a float tensor, we use the ``self.threshold`` argument to convert into integer labels. This is the case for binary and multi-label probabilities. If pred has an extra dimension as in the case of multi-class scores we perform an argmax on ``dim=1``. Args: preds: tensor containing predictions from model (probabilities, or labels) with shape ``[N, d1, d2, ...]`` target: tensor containing ground truth labels with shape ``[N, d1, d2, ...]`` ignore_index: optional int specifying a target class to ignore. If given, this class index does not contribute to the returned score, regardless of reduction method. Has no effect if given an int that is not in the range [0, num_classes-1], where num_classes is either given or derived from pred and target. By default, no index is ignored, and all classes are used. absent_score: score to use for an individual class, if no instances of the class index were present in `pred` AND no instances of the class index were present in `target`. For example, if we have 3 classes, [0, 0] for `pred`, and [0, 2] for `target`, then class 1 would be assigned the `absent_score`. threshold: Threshold value for binary or multi-label probabilities. default: 0.5 num_classes: Optionally specify the number of classes reduction: a method to reduce metric score over labels. - ``'elementwise_mean'``: takes the mean (default) - ``'sum'``: takes the sum - ``'none'``: no reduction will be applied Return: IoU score : Tensor containing single value if reduction is 'elementwise_mean', or number of classes if reduction is 'none' Example: >>> target = torch.randint(0, 2, (10, 25, 25)) >>> pred = torch.tensor(target) >>> pred[2:5, 7:13, 9:15] = 1 - pred[2:5, 7:13, 9:15] >>> iou(pred, target) tensor(0.9660) """ num_classes = get_num_classes(pred=pred, target=target, num_classes=num_classes) confmat = _confusion_matrix_update(pred, target, num_classes, threshold) return _iou_from_confmat(confmat, num_classes, ignore_index, absent_score, reduction)
def test_get_num_classes(pred, target, num_classes, expected_num_classes): assert get_num_classes(pred, target, num_classes) == expected_num_classes
def iou( pred: torch.Tensor, target: torch.Tensor, ignore_index: Optional[int] = None, absent_score: float = 0.0, num_classes: Optional[int] = None, reduction: str = 'elementwise_mean', ) -> torch.Tensor: """ Intersection over union, or Jaccard index calculation. Args: pred: Tensor containing integer predictions, with shape [N, d1, d2, ...] target: Tensor containing integer targets, with shape [N, d1, d2, ...] ignore_index: optional int specifying a target class to ignore. If given, this class index does not contribute to the returned score, regardless of reduction method. Has no effect if given an int that is not in the range [0, num_classes-1], where num_classes is either given or derived from pred and target. By default, no index is ignored, and all classes are used. absent_score: score to use for an individual class, if no instances of the class index were present in `pred` AND no instances of the class index were present in `target`. For example, if we have 3 classes, [0, 0] for `pred`, and [0, 2] for `target`, then class 1 would be assigned the `absent_score`. Default is 0.0. num_classes: Optionally specify the number of classes reduction: a method to reduce metric score over labels. - ``'elementwise_mean'``: takes the mean (default) - ``'sum'``: takes the sum - ``'none'``: no reduction will be applied Return: IoU score : Tensor containing single value if reduction is 'elementwise_mean', or number of classes if reduction is 'none' Example: >>> target = torch.randint(0, 2, (10, 25, 25)) >>> pred = torch.tensor(target) >>> pred[2:5, 7:13, 9:15] = 1 - pred[2:5, 7:13, 9:15] >>> iou(pred, target) tensor(0.9660) """ if pred.size() != target.size(): raise ValueError( f"'pred' shape ({pred.size()}) must equal 'target' shape ({target.size()})" ) if not torch.allclose(pred.float(), pred.int().float()): raise ValueError("'pred' must contain integer targets.") num_classes = get_num_classes(pred=pred, target=target, num_classes=num_classes) tps, fps, tns, fns, sups = stat_scores_multiple_classes( pred, target, num_classes) scores = torch.zeros(num_classes, device=pred.device, dtype=torch.float32) for class_idx in range(num_classes): if class_idx == ignore_index: continue tp = tps[class_idx] fp = fps[class_idx] fn = fns[class_idx] sup = sups[class_idx] # If this class is absent in the target (no support) AND absent in the pred (no true or false # positives), then use the absent_score for this class. if sup + tp + fp == 0: scores[class_idx] = absent_score continue denom = tp + fp + fn # Note that we do not need to worry about division-by-zero here since we know (sup + tp + fp != 0) from above, # which means ((tp+fn) + tp + fp != 0), which means (2tp + fp + fn != 0). Since all vars are non-negative, we # can conclude (tp + fp + fn > 0), meaning the denominator is non-zero for each class. score = tp.to(torch.float) / denom scores[class_idx] = score # Remove the ignored class index from the scores. if ignore_index is not None and ignore_index >= 0 and ignore_index < num_classes: scores = torch.cat([ scores[:ignore_index], scores[ignore_index + 1:], ]) return reduce(scores, reduction=reduction)
def stat_scores_multiple_classes( pred: torch.Tensor, target: torch.Tensor, num_classes: Optional[int] = None, argmax_dim: int = 1, reduction: str = 'none', ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ Calculates the number of true positive, false positive, true negative and false negative for each class Args: pred: prediction tensor target: target tensor num_classes: number of classes if known argmax_dim: if pred is a tensor of probabilities, this indicates the axis the argmax transformation will be applied over reduction: a method to reduce metric score over labels (default: none) Available reduction methods: - elementwise_mean: takes the mean - none: pass array - sum: add elements Return: True Positive, False Positive, True Negative, False Negative, Support Example: >>> x = torch.tensor([1, 2, 3]) >>> y = torch.tensor([0, 2, 3]) >>> tps, fps, tns, fns, sups = stat_scores_multiple_classes(x, y) >>> tps tensor([0., 0., 1., 1.]) >>> fps tensor([0., 1., 0., 0.]) >>> tns tensor([2., 2., 2., 2.]) >>> fns tensor([1., 0., 0., 0.]) >>> sups tensor([1., 0., 1., 1.]) """ if pred.ndim == target.ndim + 1: pred = to_categorical(pred, argmax_dim=argmax_dim) num_classes = get_num_classes(pred=pred, target=target, num_classes=num_classes) if pred.dtype != torch.bool: pred = pred.clamp_max(max=num_classes) if target.dtype != torch.bool: target = target.clamp_max(max=num_classes) possible_reductions = ('none', 'sum', 'elementwise_mean') if reduction not in possible_reductions: raise ValueError("reduction type %s not supported" % reduction) if reduction == 'none': pred = pred.view((-1, )).long() target = target.view((-1, )).long() tps = torch.zeros((num_classes + 1, ), device=pred.device) fps = torch.zeros((num_classes + 1, ), device=pred.device) tns = torch.zeros((num_classes + 1, ), device=pred.device) fns = torch.zeros((num_classes + 1, ), device=pred.device) sups = torch.zeros((num_classes + 1, ), device=pred.device) match_true = (pred == target).float() match_false = 1 - match_true tps.scatter_add_(0, pred, match_true) fps.scatter_add_(0, pred, match_false) fns.scatter_add_(0, target, match_false) tns = pred.size(0) - (tps + fps + fns) sups.scatter_add_(0, target, torch.ones_like(match_true)) tps = tps[:num_classes] fps = fps[:num_classes] tns = tns[:num_classes] fns = fns[:num_classes] sups = sups[:num_classes] elif reduction == 'sum' or reduction == 'elementwise_mean': count_match_true = (pred == target).sum().float() oob_tp, oob_fp, oob_tn, oob_fn, oob_sup = stat_scores( pred, target, num_classes, argmax_dim) tps = count_match_true - oob_tp fps = pred.nelement() - count_match_true - oob_fp fns = pred.nelement() - count_match_true - oob_fn tns = pred.nelement() * (num_classes + 1) - (tps + fps + fns + oob_tn) sups = pred.nelement() - oob_sup.float() if reduction == 'elementwise_mean': tps /= num_classes fps /= num_classes fns /= num_classes tns /= num_classes sups /= num_classes return tps.float(), fps.float(), tns.float(), fns.float(), sups.float()