def test_v1_5_metrics_utils():
x = torch.tensor([1, 2, 3])
with pytest.deprecated_call(match="It will be removed in v1.5.0"):
assert torch.equal(to_onehot(x), torch.Tensor([[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]).to(int))
with pytest.deprecated_call(match="It will be removed in v1.5.0"):
assert get_num_classes(torch.tensor([1, 2, 3]), torch.tensor([1, 2, 0])) == 4
x = torch.tensor([[1.1, 2.0, 3.0], [2.0, 1.0, 0.5]])
with pytest.deprecated_call(match="It will be removed in v1.5.0"):
assert torch.equal(select_topk(x, topk=2), torch.Tensor([[0, 1, 1], [1, 1, 0]]).to(torch.int32))
x = torch.tensor([[0.2, 0.5], [0.9, 0.1]])
with pytest.deprecated_call(match="It will be removed in v1.5.0"):
assert torch.equal(to_categorical(x), torch.Tensor([1, 0]).to(int))
def __multiclass_roc(
pred: torch.Tensor,
target: torch.Tensor,
sample_weight: Optional[Sequence] = None,
num_classes: Optional[int] = None,
) -> Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
"""
Computes the Receiver Operating Characteristic (ROC) for multiclass predictors.
.. warning:: Deprecated
Args:
pred: estimated probabilities
target: ground-truth labels
sample_weight: sample weights
num_classes: number of classes (default: None, computes automatically from data)
Return:
returns roc for each class.
Number of classes, false-positive rate (fpr), true-positive rate (tpr), thresholds
Example:
>>> pred = torch.tensor([[0.85, 0.05, 0.05, 0.05],
... [0.05, 0.85, 0.05, 0.05],
... [0.05, 0.05, 0.85, 0.05],
... [0.05, 0.05, 0.05, 0.85]])
>>> target = torch.tensor([0, 1, 3, 2])
>>> __multiclass_roc(pred, target) # doctest: +NORMALIZE_WHITESPACE
((tensor([0., 0., 1.]), tensor([0., 1., 1.]), tensor([1.8500, 0.8500, 0.0500])),
(tensor([0., 0., 1.]), tensor([0., 1., 1.]), tensor([1.8500, 0.8500, 0.0500])),
(tensor([0.0000, 0.3333, 1.0000]), tensor([0., 0., 1.]), tensor([1.8500, 0.8500, 0.0500])),
(tensor([0.0000, 0.3333, 1.0000]), tensor([0., 0., 1.]), tensor([1.8500, 0.8500, 0.0500])))
"""
num_classes = get_num_classes(pred, target, num_classes)
class_roc_vals = []
for c in range(num_classes):
pred_c = pred[:, c]
class_roc_vals.append(
__roc(pred=pred_c,
target=target,
sample_weight=sample_weight,
pos_label=c))
return tuple(class_roc_vals)
def stat_scores_multiple_classes(
pred: torch.Tensor,
target: torch.Tensor,
num_classes: Optional[int] = None,
argmax_dim: int = 1,
reduction: str = 'none',
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
torch.Tensor]:
"""
Calculates the number of true positive, false positive, true negative
and false negative for each class
.. warning :: Deprecated in favor of :func:`~pytorch_lightning.metrics.functional.stat_scores`
Raises:
ValueError:
If ``reduction`` is not one of ``"none"``, ``"sum"`` or ``"elementwise_mean"``.
"""
rank_zero_warn(
"This `stat_scores_multiple_classes` was deprecated in v1.2.0 in favor of"
" `from pytorch_lightning.metrics.functional import stat_scores`."
" It will be removed in v1.4.0", DeprecationWarning)
if pred.ndim == target.ndim + 1:
pred = to_categorical(pred, argmax_dim=argmax_dim)
num_classes = get_num_classes(pred=pred,
target=target,
num_classes=num_classes)
if pred.dtype != torch.bool:
pred = pred.clamp_max(max=num_classes)
if target.dtype != torch.bool:
target = target.clamp_max(max=num_classes)
possible_reductions = ('none', 'sum', 'elementwise_mean')
if reduction not in possible_reductions:
raise ValueError("reduction type %s not supported" % reduction)
if reduction == 'none':
pred = pred.view((-1, )).long()
target = target.view((-1, )).long()
tps = torch.zeros((num_classes + 1, ), device=pred.device)
fps = torch.zeros((num_classes + 1, ), device=pred.device)
fns = torch.zeros((num_classes + 1, ), device=pred.device)
sups = torch.zeros((num_classes + 1, ), device=pred.device)
match_true = (pred == target).float()
match_false = 1 - match_true
tps.scatter_add_(0, pred, match_true)
fps.scatter_add_(0, pred, match_false)
fns.scatter_add_(0, target, match_false)
tns = pred.size(0) - (tps + fps + fns)
sups.scatter_add_(0, target, torch.ones_like(match_true))
tps = tps[:num_classes]
fps = fps[:num_classes]
tns = tns[:num_classes]
fns = fns[:num_classes]
sups = sups[:num_classes]
elif reduction == 'sum' or reduction == 'elementwise_mean':
count_match_true = (pred == target).sum().float()
oob_tp, oob_fp, oob_tn, oob_fn, oob_sup = stat_scores(
pred, target, num_classes, argmax_dim)
tps = count_match_true - oob_tp
fps = pred.nelement() - count_match_true - oob_fp
fns = pred.nelement() - count_match_true - oob_fn
tns = pred.nelement() * (num_classes + 1) - (tps + fps + fns + oob_tn)
sups = pred.nelement() - oob_sup.float()
if reduction == 'elementwise_mean':
tps /= num_classes
fps /= num_classes
fns /= num_classes
tns /= num_classes
sups /= num_classes
return tps.float(), fps.float(), tns.float(), fns.float(), sups.float()
def iou(
pred: torch.Tensor,
target: torch.Tensor,
ignore_index: Optional[int] = None,
absent_score: float = 0.0,
threshold: float = 0.5,
num_classes: Optional[int] = None,
reduction: str = 'elementwise_mean',
) -> torch.Tensor:
r"""
Computes `Intersection over union, or Jaccard index calculation <https://en.wikipedia.org/wiki/Jaccard_index>`_:
.. math:: J(A,B) = \frac{|A\cap B|}{|A\cup B|}
Where: :math:`A` and :math:`B` are both tensors of the same size,
containing integer class values. They may be subject to conversion from
input data (see description below).
Note that it is different from box IoU.
If preds and target are the same shape and preds is a float tensor, we use the ``self.threshold`` argument
to convert into integer labels. This is the case for binary and multi-label probabilities.
If pred has an extra dimension as in the case of multi-class scores we
perform an argmax on ``dim=1``.
Args:
preds: tensor containing predictions from model (probabilities, or labels) with shape ``[N, d1, d2, ...]``
target: tensor containing ground truth labels with shape ``[N, d1, d2, ...]``
ignore_index: optional int specifying a target class to ignore. If given,
this class index does not contribute to the returned score, regardless
of reduction method. Has no effect if given an int that is not in the
range [0, num_classes-1], where num_classes is either given or derived
from pred and target. By default, no index is ignored, and all classes are used.
absent_score: score to use for an individual class, if no instances of
the class index were present in `pred` AND no instances of the class
index were present in `target`. For example, if we have 3 classes,
[0, 0] for `pred`, and [0, 2] for `target`, then class 1 would be
assigned the `absent_score`.
threshold:
Threshold value for binary or multi-label probabilities. default: 0.5
num_classes:
Optionally specify the number of classes
reduction: a method to reduce metric score over labels.
- ``'elementwise_mean'``: takes the mean (default)
- ``'sum'``: takes the sum
- ``'none'``: no reduction will be applied
Return:
IoU score : Tensor containing single value if reduction is
'elementwise_mean', or number of classes if reduction is 'none'
Example:
>>> target = torch.randint(0, 2, (10, 25, 25))
>>> pred = torch.tensor(target)
>>> pred[2:5, 7:13, 9:15] = 1 - pred[2:5, 7:13, 9:15]
>>> iou(pred, target)
tensor(0.9660)
"""
num_classes = get_num_classes(pred=pred, target=target, num_classes=num_classes)
confmat = _confusion_matrix_update(pred, target, num_classes, threshold)
return _iou_from_confmat(confmat, num_classes, ignore_index, absent_score, reduction)
def test_get_num_classes(pred, target, num_classes, expected_num_classes):
assert get_num_classes(pred, target, num_classes) == expected_num_classes
def iou(
pred: torch.Tensor,
target: torch.Tensor,
ignore_index: Optional[int] = None,
absent_score: float = 0.0,
num_classes: Optional[int] = None,
reduction: str = 'elementwise_mean',
) -> torch.Tensor:
"""
Intersection over union, or Jaccard index calculation.
Args:
pred: Tensor containing integer predictions, with shape [N, d1, d2, ...]
target: Tensor containing integer targets, with shape [N, d1, d2, ...]
ignore_index: optional int specifying a target class to ignore. If given, this class index does not contribute
to the returned score, regardless of reduction method. Has no effect if given an int that is not in the
range [0, num_classes-1], where num_classes is either given or derived from pred and target. By default, no
index is ignored, and all classes are used.
absent_score: score to use for an individual class, if no instances of the class index were present in
`pred` AND no instances of the class index were present in `target`. For example, if we have 3 classes,
[0, 0] for `pred`, and [0, 2] for `target`, then class 1 would be assigned the `absent_score`. Default is
0.0.
num_classes: Optionally specify the number of classes
reduction: a method to reduce metric score over labels.
- ``'elementwise_mean'``: takes the mean (default)
- ``'sum'``: takes the sum
- ``'none'``: no reduction will be applied
Return:
IoU score : Tensor containing single value if reduction is
'elementwise_mean', or number of classes if reduction is 'none'
Example:
>>> target = torch.randint(0, 2, (10, 25, 25))
>>> pred = torch.tensor(target)
>>> pred[2:5, 7:13, 9:15] = 1 - pred[2:5, 7:13, 9:15]
>>> iou(pred, target)
tensor(0.9660)
"""
if pred.size() != target.size():
raise ValueError(
f"'pred' shape ({pred.size()}) must equal 'target' shape ({target.size()})"
)
if not torch.allclose(pred.float(), pred.int().float()):
raise ValueError("'pred' must contain integer targets.")
num_classes = get_num_classes(pred=pred,
target=target,
num_classes=num_classes)
tps, fps, tns, fns, sups = stat_scores_multiple_classes(
pred, target, num_classes)
scores = torch.zeros(num_classes, device=pred.device, dtype=torch.float32)
for class_idx in range(num_classes):
if class_idx == ignore_index:
continue
tp = tps[class_idx]
fp = fps[class_idx]
fn = fns[class_idx]
sup = sups[class_idx]
# If this class is absent in the target (no support) AND absent in the pred (no true or false
# positives), then use the absent_score for this class.
if sup + tp + fp == 0:
scores[class_idx] = absent_score
continue
denom = tp + fp + fn
# Note that we do not need to worry about division-by-zero here since we know (sup + tp + fp != 0) from above,
# which means ((tp+fn) + tp + fp != 0), which means (2tp + fp + fn != 0). Since all vars are non-negative, we
# can conclude (tp + fp + fn > 0), meaning the denominator is non-zero for each class.
score = tp.to(torch.float) / denom
scores[class_idx] = score
# Remove the ignored class index from the scores.
if ignore_index is not None and ignore_index >= 0 and ignore_index < num_classes:
scores = torch.cat([
scores[:ignore_index],
scores[ignore_index + 1:],
])
return reduce(scores, reduction=reduction)
def stat_scores_multiple_classes(
pred: torch.Tensor,
target: torch.Tensor,
num_classes: Optional[int] = None,
argmax_dim: int = 1,
reduction: str = 'none',
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
torch.Tensor]:
"""
Calculates the number of true positive, false positive, true negative
and false negative for each class
Args:
pred: prediction tensor
target: target tensor
num_classes: number of classes if known
argmax_dim: if pred is a tensor of probabilities, this indicates the
axis the argmax transformation will be applied over
reduction: a method to reduce metric score over labels (default: none)
Available reduction methods:
- elementwise_mean: takes the mean
- none: pass array
- sum: add elements
Return:
True Positive, False Positive, True Negative, False Negative, Support
Example:
>>> x = torch.tensor([1, 2, 3])
>>> y = torch.tensor([0, 2, 3])
>>> tps, fps, tns, fns, sups = stat_scores_multiple_classes(x, y)
>>> tps
tensor([0., 0., 1., 1.])
>>> fps
tensor([0., 1., 0., 0.])
>>> tns
tensor([2., 2., 2., 2.])
>>> fns
tensor([1., 0., 0., 0.])
>>> sups
tensor([1., 0., 1., 1.])
"""
if pred.ndim == target.ndim + 1:
pred = to_categorical(pred, argmax_dim=argmax_dim)
num_classes = get_num_classes(pred=pred,
target=target,
num_classes=num_classes)
if pred.dtype != torch.bool:
pred = pred.clamp_max(max=num_classes)
if target.dtype != torch.bool:
target = target.clamp_max(max=num_classes)
possible_reductions = ('none', 'sum', 'elementwise_mean')
if reduction not in possible_reductions:
raise ValueError("reduction type %s not supported" % reduction)
if reduction == 'none':
pred = pred.view((-1, )).long()
target = target.view((-1, )).long()
tps = torch.zeros((num_classes + 1, ), device=pred.device)
fps = torch.zeros((num_classes + 1, ), device=pred.device)
tns = torch.zeros((num_classes + 1, ), device=pred.device)
fns = torch.zeros((num_classes + 1, ), device=pred.device)
sups = torch.zeros((num_classes + 1, ), device=pred.device)
match_true = (pred == target).float()
match_false = 1 - match_true
tps.scatter_add_(0, pred, match_true)
fps.scatter_add_(0, pred, match_false)
fns.scatter_add_(0, target, match_false)
tns = pred.size(0) - (tps + fps + fns)
sups.scatter_add_(0, target, torch.ones_like(match_true))
tps = tps[:num_classes]
fps = fps[:num_classes]
tns = tns[:num_classes]
fns = fns[:num_classes]
sups = sups[:num_classes]
elif reduction == 'sum' or reduction == 'elementwise_mean':
count_match_true = (pred == target).sum().float()
oob_tp, oob_fp, oob_tn, oob_fn, oob_sup = stat_scores(
pred, target, num_classes, argmax_dim)
tps = count_match_true - oob_tp
fps = pred.nelement() - count_match_true - oob_fp
fns = pred.nelement() - count_match_true - oob_fn
tns = pred.nelement() * (num_classes + 1) - (tps + fps + fns + oob_tn)
sups = pred.nelement() - oob_sup.float()
if reduction == 'elementwise_mean':
tps /= num_classes
fps /= num_classes
fns /= num_classes
tns /= num_classes
sups /= num_classes
return tps.float(), fps.float(), tns.float(), fns.float(), sups.float()
