def wrap_metric_fn_with_activation( metric_fn: Callable, activation: str = None, ): """Wraps model outputs for ``metric_fn` with specified ``activation``. Args: metric_fn: metric function to compute activation: activation name to use Returns: wrapped metric function with wrapped model outputs .. note:: Works only with ``metric_fn`` like ``metric_fn(outputs, targets, *args, **kwargs)``. """ activation_fn = get_activation_fn(activation) def wrapped_metric_fn(outputs: torch.Tensor, targets: torch.Tensor, *args, **kwargs): outputs = activation_fn(outputs) output = metric_fn(outputs, targets, *args, **kwargs) return output return wrapped_metric_fn
def multi_label_accuracy( outputs: torch.Tensor, targets: torch.Tensor, threshold: Union[float, torch.Tensor], activation: Optional[str] = None, ) -> torch.Tensor: """ Computes multi-label accuracy for the specified activation and threshold. Args: outputs: NxK tensor that for each of the N examples indicates the probability of the example belonging to each of the K classes, according to the model. targets: binary NxK tensort that encodes which of the K classes are associated with the N-th input (eg: a row [0, 1, 0, 1] indicates that the example is associated with classes 2 and 4) threshold: threshold for for model output activation: activation to use for model output Returns: computed multi-label accuracy """ outputs, targets, _ = preprocess_multi_label_metrics( outputs=outputs, targets=targets ) activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) outputs = (outputs > threshold).long() output = (targets.long() == outputs.long()).sum().float() / np.prod( targets.shape ) return output
def dice( outputs: torch.Tensor, targets: torch.Tensor, eps: float = 1e-7, threshold: float = None, activation: str = "Sigmoid", ): """Computes the dice metric. Args: outputs (list): a list of predicted elements targets (list): a list of elements that are to be predicted eps (float): epsilon threshold (float): threshold for outputs binarization activation (str): An torch.nn activation applied to the outputs. Must be one of ["none", "Sigmoid", "Softmax2d"] Returns: float: Dice score """ activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) if threshold is not None: outputs = (outputs > threshold).float() intersection = torch.sum(targets * outputs) union = torch.sum(targets) + torch.sum(outputs) # this looks a bit awkward but `eps * (union == 0)` term # makes sure that if I and U are both 0, than Dice == 1 # and if U != 0 and I == 0 the eps term in numerator is zeroed out # i.e. (0 + eps) / (U - 0 + eps) doesn't happen output_dice = (2 * intersection + eps * (union == 0)) / (union + eps) return output_dice
def __init__( self, target_key: str, label_key: str, filename_key: str, embedding_key: str, logit_key: str = None, class_names: List[str] = None, activation: Optional[str] = None, enable_benchmark: bool = False, benchmark_train_loader: str = None, benchmark_test_loader: str = None, benchmark_xlsx: Union[str, Path] = None, enable_doev1: bool = False, doev1_train_loaders: Union[str, List[str]] = None, doev1_test_loaders: Union[str, List[str]] = None, enable_doev2: bool = False, doev2_train_loaders: Union[str, List[str]] = None, doev2_test_loaders: Union[str, List[str]] = None, doev2_xlsx: str = None, save_dir: str = None, save_loaders: Union[str, List[str]] = None, ): super().__init__(CallbackOrder.Metric) self.filename_key = filename_key self.embedding_key = embedding_key self.labels_key = label_key self.target_key = target_key self.logit_key = logit_key self.save_dir = save_dir self.class_names = {i: name for i, name in enumerate(class_names)} self.class_ids = {name: i for i, name in enumerate(class_names)} self.activation_fn = get_activation_fn(activation) self.enable_benchmark = enable_benchmark if enable_benchmark: self.benchmark_train_loader = benchmark_train_loader self.benchmark_test_loader = benchmark_test_loader self.benchmark_index: Dict = build_benchmark_index(benchmark_xlsx) self.enable_doev1 = enable_doev1 if enable_doev1: self.doev1_train_loaders = doev1_train_loaders self.doev1_test_loaders = doev1_test_loaders self.enable_doev2 = enable_doev2 if enable_doev2: self.doev2_train_loaders = doev2_train_loaders self.doev2_test_loaders = doev2_test_loaders self.doev2_dfs: Dict[str, pd.DataFrame] = \ pd.read_excel(doev2_xlsx, sheet_name=None) self.save_loaders = save_loaders
def multi_label_metrics( outputs: torch.Tensor, targets: torch.Tensor, threshold: Union[float, torch.Tensor], activation: Optional[str] = None, eps: float = 1e-7, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ Computes multi-label precision for the specified activation and threshold. Args: outputs (torch.Tensor): NxK tensor that for each of the N examples indicates the probability of the example belonging to each of the K classes, according to the model. targets (torch.Tensor): binary NxK tensort that encodes which of the K classes are associated with the N-th input (eg: a row [0, 1, 0, 1] indicates that the example is associated with classes 2 and 4) threshold (float): threshold for for model output activation (str): activation to use for model output eps (float): epsilon to avoid zero division Extended version of https://github.com/catalyst-team/catalyst/blob/master/catalyst/utils/metrics/accuracy.py#L58 Returns: computed multi-label metrics """ outputs, targets, _ = preprocess_multi_label_metrics( outputs=outputs, targets=targets ) activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) outputs = (outputs > threshold).long() print(f'outputs.size() = {outputs.size()}') print(f'outputs = {outputs}') print(f'targets.size() = {targets.size()}') print(f'targets = {targets}') accuracy = (targets.long() == outputs.long()).sum().float() / np.prod( targets.shape ) intersection = (outputs.long() * targets.long()).sum(axis=1).float() num_predicted = outputs.long().sum(axis=1).float() num_relevant = targets.long().sum(axis=1).float() union = num_predicted + num_relevant # Precision = ({predicted items} && {relevant items}) / {predicted items} precision = intersection / (num_predicted + eps * (num_predicted == 0)) # Recall = ({predicted items} && {relevant items}) / {relevant items} recall = intersection / (num_relevant + eps * (num_relevant == 0)) # IoU = ({predicted items} && {relevant items}) / ({predicted items} || {relevant items}) iou = (intersection + eps * (union == 0)) / (union - intersection + eps) return accuracy, precision.mean(), recall.mean(), iou.mean()
def accuracy( outputs, targets, topk=(1, ), threshold: float = None, activation: str = None, ): """ Computes the accuracy. It can be used either for: 1. Multi-class task, in this case: - you can use topk. - threshold and activation are not required. - targets is a tensor: batch_size - outputs is a tensor: batch_size x num_classes - computes the accuracy@k for the specified values of k. 2. Multi-label task, in this case: - you must specify threshold and activation - topk will not be used (because of there is no method to apply top-k in multi-label classification). - outputs, targets are tensors with shape: batch_size x num_classes - targets is a tensor with binary vectors """ activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) if threshold: outputs = (outputs > threshold).long() # multi-label classification if len(targets.shape) > 1 and targets.size(1) > 1: res = (targets.long() == outputs.long()).sum().float() / np.prod( targets.shape) return [res] max_k = max(topk) batch_size = targets.size(0) if len(outputs.shape) == 1 or outputs.shape[1] == 1: pred = outputs.t() else: _, pred = outputs.topk(max_k, 1, True, True) pred = pred.t() correct = pred.eq(targets.long().view(1, -1).expand_as(pred)) res = [] for k in topk: correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) res.append(correct_k.mul_(1.0 / batch_size)) return res
def iou( outputs: torch.Tensor, targets: torch.Tensor, # values are discarded, only None check # used for compatibility with BatchMetricCallback classes: List[str] = None, eps: float = 1e-7, threshold: float = None, activation: str = "Sigmoid", ) -> torch.Tensor: """ Args: outputs (torch.Tensor): A list of predicted elements targets (torch.Tensor): A list of elements that are to be predicted classes (List[str]): if classes are specified we reduce across all dims except channels eps (float): epsilon to avoid zero division threshold (float): threshold for outputs binarization activation (str): An torch.nn activation applied to the outputs. Must be one of ["none", "Sigmoid", "Softmax2d"] Returns: Union[float, List[float]]: IoU (Jaccard) score(s) """ activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) if threshold is not None: outputs = (outputs > threshold).float() # TODO: fix classes issue # ! fix backward compatibility if classes is not None: # if classes are specified we reduce across all dims except channels sum_strategy = partial(torch.sum, dim=[0, 2, 3]) else: sum_strategy = torch.sum intersection = sum_strategy(targets * outputs) union = sum_strategy(targets) + sum_strategy(outputs) # this looks a bit awkward but `eps * (union == 0)` term # makes sure that if I and U are both 0, than IoU == 1 # and if U != 0 and I == 0 the eps term in numerator is zeroed out # i.e. (0 + eps) / (U - 0 + eps) doesn't happen output_iou = (intersection + eps * (union == 0)) / ( union - intersection + eps ) return output_iou
def dice( outputs: T, targets: T, eps: float = 1e-7, threshold: float = None, activation: str = "Sigmoid", ) -> T: activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) if threshold is not None: outputs = (outputs > threshold).float() intersection = torch.sum(targets * outputs) union = torch.sum(targets) + torch.sum(outputs) score = (2 * intersection + eps * (union == 0)) / (union + eps) return score
def f1_score( outputs: torch.Tensor, targets: torch.Tensor, beta: float = 1.0, eps: float = 1e-7, threshold: float = None, activation: str = "Sigmoid", ): """ Args: outputs: A list of predicted elements targets: A list of elements that are to be predicted eps: epsilon to avoid zero division beta: beta param for f_score threshold: threshold for outputs binarization activation: An torch.nn activation applied to the outputs. Must be one of ["none", "Sigmoid", "Softmax2d"] Returns: float: F_1 score """ activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) if threshold is not None: outputs = (outputs > threshold).float() true_positive = torch.sum(targets * outputs) false_positive = torch.sum(outputs) - true_positive false_negative = torch.sum(targets) - true_positive precision_plus_recall = ( (1 + beta ** 2) * true_positive + beta ** 2 * false_negative + false_positive + eps ) score = ((1 + beta ** 2) * true_positive + eps) / precision_plus_recall return score
def accuracy( outputs: torch.Tensor, targets: torch.Tensor, topk: Sequence[int] = (1, ), activation: Optional[str] = None, ) -> Sequence[torch.Tensor]: """ Computes multi-class accuracy@topk for the specified values of `topk`. Args: outputs: model outputs, logits with shape [bs; num_classes] targets: ground truth, labels with shape [bs; 1] activation: activation to use for model output topk: `topk` for accuracy@topk computing Returns: list with computed accuracy@topk """ activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) max_k = max(topk) batch_size = targets.size(0) if len(outputs.shape) == 1 or outputs.shape[1] == 1: # binary accuracy pred = outputs.t() else: # multi-class accuracy _, pred = outputs.topk(max_k, 1, True, True) # noqa: WPS425 pred = pred.t() correct = pred.eq(targets.long().view(1, -1).expand_as(pred)) output = [] for k in topk: correct_k = (correct[:k].contiguous().view(-1).float().sum( 0, keepdim=True)) output.append(correct_k.mul_(1.0 / batch_size)) return output
def tversky( outputs: T, targets: T, alpha: float = 0.7, eps: float = 1e-7, threshold: float = None, activation: str = "Sigmoid", ) -> T: activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) if threshold is not None: outputs = (outputs > threshold).float() tp = torch.sum(targets * outputs) fn = torch.sum(targets * (1 - outputs)) fp = torch.sum((1 - targets) * outputs) dn = tp + alpha * fn + (1 - alpha) * fp score = (tp + eps * (dn == 0)) / (dn + eps) return score
def __init__( self, metric_names: List[str], meter_list: List, input_key: str = "targets", output_key: str = "logits", class_names: List[str] = None, num_classes: int = 2, activation: str = "Sigmoid", ): """ Args: metric_names: of metrics to print Make sure that they are in the same order that metrics are outputted by the meters in `meter_list` meter_list: List of meters.meter.Meter instances len(meter_list) == num_classes input_key: input key to use for metric calculation specifies our ``y_true``. output_key: output key to use for metric calculation; specifies our ``y_pred`` class_names: class names to display in the logs. If None, defaults to indices for each class, starting from 0. num_classes: Number of classes; must be > 1 activation: An torch.nn activation applied to the logits. Must be one of ['none', 'Sigmoid', 'Softmax2d'] """ super().__init__(CallbackOrder.metric) self.metric_names = metric_names self.meters = meter_list self.input_key = input_key self.output_key = output_key self.class_names = class_names self.num_classes = num_classes self.activation = activation self.activation_fn = get_activation_fn(self.activation)
def accuracy( outputs: torch.Tensor, targets: torch.Tensor, topk: Tuple = (1, ), threshold: float = None, activation: str = None, ): """ Computes the accuracy. It can be used either for: 1. Multi-class task, in this case: - you can use topk. - threshold and activation are not required. - targets is a tensor: batch_size - outputs is a tensor: batch_size x num_classes - computes the accuracy@k for the specified values of k. 2. Multi-label task, in this case: - you must specify threshold and activation - topk will not be used (because of there is no method to apply top-k in multi-label classification). - outputs, targets are tensors with shape: batch_size x num_classes - targets is a tensor with binary vectors Args: outputs (torch.Tensor): model outputs, logits targets (torch.Tensor): ground truth, labels topk (tuple): tuple with specified `N` for top`N` accuracy computing threshold (float): threshold for outputs activation (str): activation for outputs Returns: computed topK accuracy """ activation_fn = get_activation_fn(activation) outputs = activation_fn(outputs) if threshold: outputs = (outputs > threshold).long() # TODO: move to separate function # multi-label classification if len(targets.shape) > 1 and targets.size(1) > 1: output = (targets.long() == outputs.long()).sum().float() / np.prod( targets.shape) return [output] max_k = max(topk) batch_size = targets.size(0) if len(outputs.shape) == 1 or outputs.shape[1] == 1: pred = outputs.t() else: _, pred = outputs.topk(max_k, 1, True, True) # noqa: WPS425 pred = pred.t() correct = pred.eq(targets.long().view(1, -1).expand_as(pred)) output = [] for k in topk: correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) output.append(correct_k.mul_(1.0 / batch_size)) return output