def test_v1_3_0_deprecated_metrics(): from pytorch_lightning.metrics.functional.classification import to_onehot with pytest.deprecated_call(match='will be removed in v1.3'): to_onehot(torch.tensor([1, 2, 3])) from pytorch_lightning.metrics.functional.classification import to_categorical with pytest.deprecated_call(match='will be removed in v1.3'): to_categorical(torch.tensor([[0.2, 0.5], [0.9, 0.1]])) from pytorch_lightning.metrics.functional.classification import get_num_classes with pytest.deprecated_call(match='will be removed in v1.3'): get_num_classes(pred=torch.tensor([0, 1]), target=torch.tensor([1, 1])) x_binary = torch.tensor([0, 1, 2, 3]) y_binary = torch.tensor([0, 1, 2, 3]) from pytorch_lightning.metrics.functional.classification import roc with pytest.deprecated_call(match='will be removed in v1.3'): roc(pred=x_binary, target=y_binary) from pytorch_lightning.metrics.functional.classification import _roc with pytest.deprecated_call(match='will be removed in v1.3'): _roc(pred=x_binary, target=y_binary) x_multy = torch.tensor([ [0.85, 0.05, 0.05, 0.05], [0.05, 0.85, 0.05, 0.05], [0.05, 0.05, 0.85, 0.05], [0.05, 0.05, 0.05, 0.85], ]) y_multy = torch.tensor([0, 1, 3, 2]) from pytorch_lightning.metrics.functional.classification import multiclass_roc with pytest.deprecated_call(match='will be removed in v1.3'): multiclass_roc(pred=x_multy, target=y_multy) from pytorch_lightning.metrics.functional.classification import average_precision with pytest.deprecated_call(match='will be removed in v1.3'): average_precision(pred=x_binary, target=y_binary) from pytorch_lightning.metrics.functional.classification import precision_recall_curve with pytest.deprecated_call(match='will be removed in v1.3'): precision_recall_curve(pred=x_binary, target=y_binary) from pytorch_lightning.metrics.functional.classification import multiclass_precision_recall_curve with pytest.deprecated_call(match='will be removed in v1.3'): multiclass_precision_recall_curve(pred=x_multy, target=y_multy) from pytorch_lightning.metrics.functional.reduction import reduce with pytest.deprecated_call(match='will be removed in v1.3'): reduce(torch.tensor([0, 1, 1, 0]), 'sum') from pytorch_lightning.metrics.functional.reduction import class_reduce with pytest.deprecated_call(match='will be removed in v1.3'): class_reduce( torch.randint(1, 10, (50, )).float(), torch.randint(10, 20, (50, )).float(), torch.randint(1, 100, (50, )).float())
def precision_recall( pred: torch.Tensor, target: torch.Tensor, num_classes: Optional[int] = None, class_reduction: str = 'micro', return_support: bool = False, return_state: bool = False) -> Tuple[torch.Tensor, torch.Tensor]: """ Computes precision and recall for different thresholds Args: pred: estimated probabilities target: ground-truth labels num_classes: number of classes class_reduction: method to reduce metric score over labels - ``'micro'``: calculate metrics globally (default) - ``'macro'``: calculate metrics for each label, and find their unweighted mean. - ``'weighted'``: calculate metrics for each label, and find their weighted mean. - ``'none'``: returns calculated metric per class return_support: returns the support for each class, need for fbeta/f1 calculations return_state: returns a internal state that can be ddp reduced before doing the final calculation Return: Tensor with precision and recall Example: >>> x = torch.tensor([0, 1, 2, 3]) >>> y = torch.tensor([0, 2, 2, 2]) >>> precision_recall(x, y, class_reduction='macro') (tensor(0.5000), tensor(0.3333)) """ tps, fps, tns, fns, sups = stat_scores_multiple_classes( pred=pred, target=target, num_classes=num_classes) precision = class_reduce(tps, tps + fps, sups, class_reduction=class_reduction) recall = class_reduce(tps, tps + fns, sups, class_reduction=class_reduction) if return_state: return {'tps': tps, 'fps': fps, 'fns': fns, 'sups': sups} if return_support: return precision, recall, sups return precision, recall
def test_class_reduce(): num = torch.randint(1, 10, (100,)).float() denom = torch.randint(10, 20, (100,)).float() weights = torch.randint(1, 100, (100,)).float() assert torch.allclose(class_reduce(num, denom, weights, 'micro'), torch.sum(num) / torch.sum(denom)) assert torch.allclose(class_reduce(num, denom, weights, 'macro'), torch.mean(num / denom)) assert torch.allclose(class_reduce(num, denom, weights, 'weighted'), torch.sum(num / denom * (weights / torch.sum(weights)))) assert torch.allclose(class_reduce(num, denom, weights, 'none'), num / denom)
def fbeta_score( pred: torch.Tensor, target: torch.Tensor, beta: float, num_classes: Optional[int] = None, class_reduction: str = 'micro', ) -> torch.Tensor: """ Computes the F-beta score which is a weighted harmonic mean of precision and recall. It ranges between 1 and 0, where 1 is perfect and the worst value is 0. Args: pred: estimated probabilities target: ground-truth labels beta: weights recall when combining the score. beta < 1: more weight to precision. beta > 1 more weight to recall beta = 0: only precision beta -> inf: only recall num_classes: number of classes class_reduction: method to reduce metric score over labels - ``'micro'``: calculate metrics globally (default) - ``'macro'``: calculate metrics for each label, and find their unweighted mean. - ``'weighted'``: calculate metrics for each label, and find their weighted mean. - ``'none'``: returns calculated metric per class Return: Tensor with the value of F-score. It is a value between 0-1. Example: >>> x = torch.tensor([0, 1, 2, 3]) >>> y = torch.tensor([0, 1, 2, 2]) >>> fbeta_score(x, y, 0.2) tensor(0.7500) """ # We need to differentiate at which point to do class reduction intermidiate_reduction = 'none' if class_reduction != "micro" else 'micro' prec, rec, sups = precision_recall(pred=pred, target=target, num_classes=num_classes, class_reduction=intermidiate_reduction, return_support=True) num = (1 + beta**2) * prec * rec denom = ((beta**2) * prec + rec) if intermidiate_reduction == 'micro': return torch.sum(num) / torch.sum(denom) return class_reduce(num, denom, sups, class_reduction=class_reduction)
def compute(self, data: Any, output: Any): """ tps, fps, fns, sups needs to be synced before we do any calculations """ tps, fps, fns, sups = output['tps'], output['fps'], output[ 'fns'], output['sups'] intermidiate_reduction = 'none' if self.class_reduction != "micro" else 'micro' precision = class_reduce(tps, tps + fps, sups, class_reduction=intermidiate_reduction) recall = class_reduce(tps, tps + fns, sups, class_reduction=intermidiate_reduction) num = (1 + self.beta**2) * precision * recall denom = ((self.beta**2) * precision + recall) if intermidiate_reduction == 'micro': return torch.sum(num) / torch.sum(denom) return class_reduce(num, denom, sups, class_reduction=self.class_reduction)
def _fbeta_compute( true_positives: torch.Tensor, predicted_positives: torch.Tensor, actual_positives: torch.Tensor, beta: float = 1.0, average: str = "micro" ) -> torch.Tensor: if average == "micro": precision = true_positives.sum().float() / predicted_positives.sum() recall = true_positives.sum().float() / actual_positives.sum() else: precision = true_positives.float() / predicted_positives recall = true_positives.float() / actual_positives num = (1 + beta ** 2) * precision * recall denom = beta ** 2 * precision + recall return class_reduce(num, denom, weights=actual_positives, class_reduction=average)
def compute(self): """ Computes accuracy over state. """ if self.average == 'micro': precision = self.true_positives.sum().float() / ( self.predicted_positives.sum()) recall = self.true_positives.sum().float() / ( self.actual_positives.sum()) elif self.average == 'macro': precision = self.true_positives.float() / ( self.predicted_positives) recall = self.true_positives.float() / (self.actual_positives) num = (1 + self.beta**2) * precision * recall denom = self.beta**2 * precision + recall return class_reduce(num=num, denom=denom, weights=None, class_reduction='macro')
def accuracy( pred: torch.Tensor, target: torch.Tensor, num_classes: Optional[int] = None, class_reduction: str = 'micro', return_state: bool = False ) -> torch.Tensor: """ Computes the accuracy classification score Args: pred: predicted labels target: ground truth labels num_classes: number of classes class_reduction: method to reduce metric score over labels - ``'micro'``: calculate metrics globally (default) - ``'macro'``: calculate metrics for each label, and find their unweighted mean. - ``'weighted'``: calculate metrics for each label, and find their weighted mean. - ``'none'``: returns calculated metric per class return_state: returns a internal state that can be ddp reduced before doing the final calculation Return: A Tensor with the accuracy score. Example: >>> x = torch.tensor([0, 1, 2, 3]) >>> y = torch.tensor([0, 1, 2, 2]) >>> accuracy(x, y) tensor(0.7500) """ tps, fps, tns, fns, sups = stat_scores_multiple_classes( pred=pred, target=target, num_classes=num_classes) if return_state: return {'tps': tps, 'sups': sups} return class_reduce(tps, sups, sups, class_reduction=class_reduction)
def accuracy(pred: torch.Tensor, target: torch.Tensor, num_classes: Optional[int] = None, class_reduction: str = 'micro') -> torch.Tensor: """ Computes the accuracy classification score Args: pred: predicted labels target: ground truth labels num_classes: number of classes class_reduction: method to reduce metric score over labels - ``'micro'``: calculate metrics globally (default) - ``'macro'``: calculate metrics for each label, and find their unweighted mean. - ``'weighted'``: calculate metrics for each label, and find their weighted mean. - ``'none'``: returns calculated metric per class Return: A Tensor with the accuracy score. Example: >>> x = torch.tensor([0, 1, 2, 3]) >>> y = torch.tensor([0, 1, 2, 2]) >>> accuracy(x, y) tensor(0.7500) """ if not (target > 0).any() and num_classes is None: raise RuntimeError( "cannot infer num_classes when target is all zero. If you input all-zero tensor, please specify 'num_classes=' value." ) tps, fps, tns, fns, sups = stat_scores_multiple_classes( pred=pred, target=target, num_classes=num_classes) return class_reduce(tps, sups, sups, class_reduction=class_reduction)
def compute(self, data: Any, output: Any): tps, sups = output['tps'], output['sups'] return class_reduce(tps, sups, sups, class_reduction=self.class_reduction)