def test_v1_3_0_deprecated_metrics():
from pytorch_lightning.metrics.functional.classification import to_onehot
with pytest.deprecated_call(match='will be removed in v1.3'):
to_onehot(torch.tensor([1, 2, 3]))
from pytorch_lightning.metrics.functional.classification import to_categorical
with pytest.deprecated_call(match='will be removed in v1.3'):
to_categorical(torch.tensor([[0.2, 0.5], [0.9, 0.1]]))
from pytorch_lightning.metrics.functional.classification import get_num_classes
with pytest.deprecated_call(match='will be removed in v1.3'):
get_num_classes(pred=torch.tensor([0, 1]), target=torch.tensor([1, 1]))
x_binary = torch.tensor([0, 1, 2, 3])
y_binary = torch.tensor([0, 1, 2, 3])
from pytorch_lightning.metrics.functional.classification import roc
with pytest.deprecated_call(match='will be removed in v1.3'):
roc(pred=x_binary, target=y_binary)
from pytorch_lightning.metrics.functional.classification import _roc
with pytest.deprecated_call(match='will be removed in v1.3'):
_roc(pred=x_binary, target=y_binary)
x_multy = torch.tensor([
[0.85, 0.05, 0.05, 0.05],
[0.05, 0.85, 0.05, 0.05],
[0.05, 0.05, 0.85, 0.05],
[0.05, 0.05, 0.05, 0.85],
])
y_multy = torch.tensor([0, 1, 3, 2])
from pytorch_lightning.metrics.functional.classification import multiclass_roc
with pytest.deprecated_call(match='will be removed in v1.3'):
multiclass_roc(pred=x_multy, target=y_multy)
from pytorch_lightning.metrics.functional.classification import average_precision
with pytest.deprecated_call(match='will be removed in v1.3'):
average_precision(pred=x_binary, target=y_binary)
from pytorch_lightning.metrics.functional.classification import precision_recall_curve
with pytest.deprecated_call(match='will be removed in v1.3'):
precision_recall_curve(pred=x_binary, target=y_binary)
from pytorch_lightning.metrics.functional.classification import multiclass_precision_recall_curve
with pytest.deprecated_call(match='will be removed in v1.3'):
multiclass_precision_recall_curve(pred=x_multy, target=y_multy)
from pytorch_lightning.metrics.functional.reduction import reduce
with pytest.deprecated_call(match='will be removed in v1.3'):
reduce(torch.tensor([0, 1, 1, 0]), 'sum')
from pytorch_lightning.metrics.functional.reduction import class_reduce
with pytest.deprecated_call(match='will be removed in v1.3'):
class_reduce(
torch.randint(1, 10, (50, )).float(),
torch.randint(10, 20, (50, )).float(),
torch.randint(1, 100, (50, )).float())
def precision_recall(
pred: torch.Tensor,
target: torch.Tensor,
num_classes: Optional[int] = None,
class_reduction: str = 'micro',
return_support: bool = False,
return_state: bool = False) -> Tuple[torch.Tensor, torch.Tensor]:
"""
Computes precision and recall for different thresholds
Args:
pred: estimated probabilities
target: ground-truth labels
num_classes: number of classes
class_reduction: method to reduce metric score over labels
- ``'micro'``: calculate metrics globally (default)
- ``'macro'``: calculate metrics for each label, and find their unweighted mean.
- ``'weighted'``: calculate metrics for each label, and find their weighted mean.
- ``'none'``: returns calculated metric per class
return_support: returns the support for each class, need for fbeta/f1 calculations
return_state: returns a internal state that can be ddp reduced
before doing the final calculation
Return:
Tensor with precision and recall
Example:
>>> x = torch.tensor([0, 1, 2, 3])
>>> y = torch.tensor([0, 2, 2, 2])
>>> precision_recall(x, y, class_reduction='macro')
(tensor(0.5000), tensor(0.3333))
"""
tps, fps, tns, fns, sups = stat_scores_multiple_classes(
pred=pred, target=target, num_classes=num_classes)
precision = class_reduce(tps,
tps + fps,
sups,
class_reduction=class_reduction)
recall = class_reduce(tps,
tps + fns,
sups,
class_reduction=class_reduction)
if return_state:
return {'tps': tps, 'fps': fps, 'fns': fns, 'sups': sups}
if return_support:
return precision, recall, sups
return precision, recall
def test_class_reduce():
num = torch.randint(1, 10, (100,)).float()
denom = torch.randint(10, 20, (100,)).float()
weights = torch.randint(1, 100, (100,)).float()
assert torch.allclose(class_reduce(num, denom, weights, 'micro'),
torch.sum(num) / torch.sum(denom))
assert torch.allclose(class_reduce(num, denom, weights, 'macro'),
torch.mean(num / denom))
assert torch.allclose(class_reduce(num, denom, weights, 'weighted'),
torch.sum(num / denom * (weights / torch.sum(weights))))
assert torch.allclose(class_reduce(num, denom, weights, 'none'),
num / denom)
def fbeta_score(
pred: torch.Tensor,
target: torch.Tensor,
beta: float,
num_classes: Optional[int] = None,
class_reduction: str = 'micro',
) -> torch.Tensor:
"""
Computes the F-beta score which is a weighted harmonic mean of precision and recall.
It ranges between 1 and 0, where 1 is perfect and the worst value is 0.
Args:
pred: estimated probabilities
target: ground-truth labels
beta: weights recall when combining the score.
beta < 1: more weight to precision.
beta > 1 more weight to recall
beta = 0: only precision
beta -> inf: only recall
num_classes: number of classes
class_reduction: method to reduce metric score over labels
- ``'micro'``: calculate metrics globally (default)
- ``'macro'``: calculate metrics for each label, and find their unweighted mean.
- ``'weighted'``: calculate metrics for each label, and find their weighted mean.
- ``'none'``: returns calculated metric per class
Return:
Tensor with the value of F-score. It is a value between 0-1.
Example:
>>> x = torch.tensor([0, 1, 2, 3])
>>> y = torch.tensor([0, 1, 2, 2])
>>> fbeta_score(x, y, 0.2)
tensor(0.7500)
"""
# We need to differentiate at which point to do class reduction
intermidiate_reduction = 'none' if class_reduction != "micro" else 'micro'
prec, rec, sups = precision_recall(pred=pred,
target=target,
num_classes=num_classes,
class_reduction=intermidiate_reduction,
return_support=True)
num = (1 + beta**2) * prec * rec
denom = ((beta**2) * prec + rec)
if intermidiate_reduction == 'micro':
return torch.sum(num) / torch.sum(denom)
return class_reduce(num, denom, sups, class_reduction=class_reduction)
def compute(self, data: Any, output: Any):
""" tps, fps, fns, sups needs to be synced before we do any calculations """
tps, fps, fns, sups = output['tps'], output['fps'], output[
'fns'], output['sups']
intermidiate_reduction = 'none' if self.class_reduction != "micro" else 'micro'
precision = class_reduce(tps,
tps + fps,
sups,
class_reduction=intermidiate_reduction)
recall = class_reduce(tps,
tps + fns,
sups,
class_reduction=intermidiate_reduction)
num = (1 + self.beta**2) * precision * recall
denom = ((self.beta**2) * precision + recall)
if intermidiate_reduction == 'micro':
return torch.sum(num) / torch.sum(denom)
return class_reduce(num,
denom,
sups,
class_reduction=self.class_reduction)
def _fbeta_compute(
true_positives: torch.Tensor,
predicted_positives: torch.Tensor,
actual_positives: torch.Tensor,
beta: float = 1.0,
average: str = "micro"
) -> torch.Tensor:
if average == "micro":
precision = true_positives.sum().float() / predicted_positives.sum()
recall = true_positives.sum().float() / actual_positives.sum()
else:
precision = true_positives.float() / predicted_positives
recall = true_positives.float() / actual_positives
num = (1 + beta ** 2) * precision * recall
denom = beta ** 2 * precision + recall
return class_reduce(num, denom, weights=actual_positives, class_reduction=average)
def compute(self):
"""
Computes accuracy over state.
"""
if self.average == 'micro':
precision = self.true_positives.sum().float() / (
self.predicted_positives.sum())
recall = self.true_positives.sum().float() / (
self.actual_positives.sum())
elif self.average == 'macro':
precision = self.true_positives.float() / (
self.predicted_positives)
recall = self.true_positives.float() / (self.actual_positives)
num = (1 + self.beta**2) * precision * recall
denom = self.beta**2 * precision + recall
return class_reduce(num=num,
denom=denom,
weights=None,
class_reduction='macro')
def accuracy(
pred: torch.Tensor,
target: torch.Tensor,
num_classes: Optional[int] = None,
class_reduction: str = 'micro',
return_state: bool = False
) -> torch.Tensor:
"""
Computes the accuracy classification score
Args:
pred: predicted labels
target: ground truth labels
num_classes: number of classes
class_reduction: method to reduce metric score over labels
- ``'micro'``: calculate metrics globally (default)
- ``'macro'``: calculate metrics for each label, and find their unweighted mean.
- ``'weighted'``: calculate metrics for each label, and find their weighted mean.
- ``'none'``: returns calculated metric per class
return_state: returns a internal state that can be ddp reduced
before doing the final calculation
Return:
A Tensor with the accuracy score.
Example:
>>> x = torch.tensor([0, 1, 2, 3])
>>> y = torch.tensor([0, 1, 2, 2])
>>> accuracy(x, y)
tensor(0.7500)
"""
tps, fps, tns, fns, sups = stat_scores_multiple_classes(
pred=pred, target=target, num_classes=num_classes)
if return_state:
return {'tps': tps, 'sups': sups}
return class_reduce(tps, sups, sups, class_reduction=class_reduction)
def accuracy(pred: torch.Tensor,
target: torch.Tensor,
num_classes: Optional[int] = None,
class_reduction: str = 'micro') -> torch.Tensor:
"""
Computes the accuracy classification score
Args:
pred: predicted labels
target: ground truth labels
num_classes: number of classes
class_reduction: method to reduce metric score over labels
- ``'micro'``: calculate metrics globally (default)
- ``'macro'``: calculate metrics for each label, and find their unweighted mean.
- ``'weighted'``: calculate metrics for each label, and find their weighted mean.
- ``'none'``: returns calculated metric per class
Return:
A Tensor with the accuracy score.
Example:
>>> x = torch.tensor([0, 1, 2, 3])
>>> y = torch.tensor([0, 1, 2, 2])
>>> accuracy(x, y)
tensor(0.7500)
"""
if not (target > 0).any() and num_classes is None:
raise RuntimeError(
"cannot infer num_classes when target is all zero. If you input all-zero tensor, please specify 'num_classes=' value."
)
tps, fps, tns, fns, sups = stat_scores_multiple_classes(
pred=pred, target=target, num_classes=num_classes)
return class_reduce(tps, sups, sups, class_reduction=class_reduction)
def compute(self, data: Any, output: Any):
tps, sups = output['tps'], output['sups']
return class_reduce(tps,
sups,
sups,
class_reduction=self.class_reduction)
