def wrap_metric_fn_with_activation(
metric_fn: Callable,
activation: str = None,
):
"""Wraps model outputs for ``metric_fn` with specified ``activation``.
Args:
metric_fn: metric function to compute
activation: activation name to use
Returns:
wrapped metric function with wrapped model outputs
.. note::
Works only with ``metric_fn`` like
``metric_fn(outputs, targets, *args, **kwargs)``.
"""
activation_fn = get_activation_fn(activation)
def wrapped_metric_fn(outputs: torch.Tensor, targets: torch.Tensor, *args,
**kwargs):
outputs = activation_fn(outputs)
output = metric_fn(outputs, targets, *args, **kwargs)
return output
return wrapped_metric_fn
def multi_label_accuracy(
outputs: torch.Tensor,
targets: torch.Tensor,
threshold: Union[float, torch.Tensor],
activation: Optional[str] = None,
) -> torch.Tensor:
"""
Computes multi-label accuracy for the specified activation and threshold.
Args:
outputs: NxK tensor that for each of the N examples
indicates the probability of the example belonging to each of
the K classes, according to the model.
targets: binary NxK tensort that encodes which of the K
classes are associated with the N-th input
(eg: a row [0, 1, 0, 1] indicates that the example is
associated with classes 2 and 4)
threshold: threshold for for model output
activation: activation to use for model output
Returns:
computed multi-label accuracy
"""
outputs, targets, _ = preprocess_multi_label_metrics(
outputs=outputs, targets=targets
)
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
outputs = (outputs > threshold).long()
output = (targets.long() == outputs.long()).sum().float() / np.prod(
targets.shape
)
return output
def dice(
outputs: torch.Tensor,
targets: torch.Tensor,
eps: float = 1e-7,
threshold: float = None,
activation: str = "Sigmoid",
):
"""Computes the dice metric.
Args:
outputs (list): a list of predicted elements
targets (list): a list of elements that are to be predicted
eps (float): epsilon
threshold (float): threshold for outputs binarization
activation (str): An torch.nn activation applied to the outputs.
Must be one of ["none", "Sigmoid", "Softmax2d"]
Returns:
float: Dice score
"""
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
if threshold is not None:
outputs = (outputs > threshold).float()
intersection = torch.sum(targets * outputs)
union = torch.sum(targets) + torch.sum(outputs)
# this looks a bit awkward but `eps * (union == 0)` term
# makes sure that if I and U are both 0, than Dice == 1
# and if U != 0 and I == 0 the eps term in numerator is zeroed out
# i.e. (0 + eps) / (U - 0 + eps) doesn't happen
output_dice = (2 * intersection + eps * (union == 0)) / (union + eps)
return output_dice
def __init__(
self,
target_key: str,
label_key: str,
filename_key: str,
embedding_key: str,
logit_key: str = None,
class_names: List[str] = None,
activation: Optional[str] = None,
enable_benchmark: bool = False,
benchmark_train_loader: str = None,
benchmark_test_loader: str = None,
benchmark_xlsx: Union[str, Path] = None,
enable_doev1: bool = False,
doev1_train_loaders: Union[str, List[str]] = None,
doev1_test_loaders: Union[str, List[str]] = None,
enable_doev2: bool = False,
doev2_train_loaders: Union[str, List[str]] = None,
doev2_test_loaders: Union[str, List[str]] = None,
doev2_xlsx: str = None,
save_dir: str = None,
save_loaders: Union[str, List[str]] = None,
):
super().__init__(CallbackOrder.Metric)
self.filename_key = filename_key
self.embedding_key = embedding_key
self.labels_key = label_key
self.target_key = target_key
self.logit_key = logit_key
self.save_dir = save_dir
self.class_names = {i: name for i, name in enumerate(class_names)}
self.class_ids = {name: i for i, name in enumerate(class_names)}
self.activation_fn = get_activation_fn(activation)
self.enable_benchmark = enable_benchmark
if enable_benchmark:
self.benchmark_train_loader = benchmark_train_loader
self.benchmark_test_loader = benchmark_test_loader
self.benchmark_index: Dict = build_benchmark_index(benchmark_xlsx)
self.enable_doev1 = enable_doev1
if enable_doev1:
self.doev1_train_loaders = doev1_train_loaders
self.doev1_test_loaders = doev1_test_loaders
self.enable_doev2 = enable_doev2
if enable_doev2:
self.doev2_train_loaders = doev2_train_loaders
self.doev2_test_loaders = doev2_test_loaders
self.doev2_dfs: Dict[str, pd.DataFrame] = \
pd.read_excel(doev2_xlsx, sheet_name=None)
self.save_loaders = save_loaders
def multi_label_metrics(
outputs: torch.Tensor,
targets: torch.Tensor,
threshold: Union[float, torch.Tensor],
activation: Optional[str] = None,
eps: float = 1e-7,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Computes multi-label precision for the specified activation and threshold.
Args:
outputs (torch.Tensor): NxK tensor that for each of the N examples
indicates the probability of the example belonging to each of
the K classes, according to the model.
targets (torch.Tensor): binary NxK tensort that encodes which of the K
classes are associated with the N-th input
(eg: a row [0, 1, 0, 1] indicates that the example is
associated with classes 2 and 4)
threshold (float): threshold for for model output
activation (str): activation to use for model output
eps (float): epsilon to avoid zero division
Extended version of
https://github.com/catalyst-team/catalyst/blob/master/catalyst/utils/metrics/accuracy.py#L58
Returns:
computed multi-label metrics
"""
outputs, targets, _ = preprocess_multi_label_metrics(
outputs=outputs, targets=targets
)
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
outputs = (outputs > threshold).long()
print(f'outputs.size() = {outputs.size()}')
print(f'outputs = {outputs}')
print(f'targets.size() = {targets.size()}')
print(f'targets = {targets}')
accuracy = (targets.long() == outputs.long()).sum().float() / np.prod(
targets.shape
)
intersection = (outputs.long() * targets.long()).sum(axis=1).float()
num_predicted = outputs.long().sum(axis=1).float()
num_relevant = targets.long().sum(axis=1).float()
union = num_predicted + num_relevant
# Precision = ({predicted items} && {relevant items}) / {predicted items}
precision = intersection / (num_predicted + eps * (num_predicted == 0))
# Recall = ({predicted items} && {relevant items}) / {relevant items}
recall = intersection / (num_relevant + eps * (num_relevant == 0))
# IoU = ({predicted items} && {relevant items}) / ({predicted items} || {relevant items})
iou = (intersection + eps * (union == 0)) / (union - intersection + eps)
return accuracy, precision.mean(), recall.mean(), iou.mean()
def accuracy(
outputs,
targets,
topk=(1, ),
threshold: float = None,
activation: str = None,
):
"""
Computes the accuracy.
It can be used either for:
1. Multi-class task, in this case:
- you can use topk.
- threshold and activation are not required.
- targets is a tensor: batch_size
- outputs is a tensor: batch_size x num_classes
- computes the accuracy@k for the specified values of k.
2. Multi-label task, in this case:
- you must specify threshold and activation
- topk will not be used
(because of there is no method to apply top-k in
multi-label classification).
- outputs, targets are tensors with shape: batch_size x num_classes
- targets is a tensor with binary vectors
"""
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
if threshold:
outputs = (outputs > threshold).long()
# multi-label classification
if len(targets.shape) > 1 and targets.size(1) > 1:
res = (targets.long() == outputs.long()).sum().float() / np.prod(
targets.shape)
return [res]
max_k = max(topk)
batch_size = targets.size(0)
if len(outputs.shape) == 1 or outputs.shape[1] == 1:
pred = outputs.t()
else:
_, pred = outputs.topk(max_k, 1, True, True)
pred = pred.t()
correct = pred.eq(targets.long().view(1, -1).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
res.append(correct_k.mul_(1.0 / batch_size))
return res
def iou(
outputs: torch.Tensor,
targets: torch.Tensor,
# values are discarded, only None check
# used for compatibility with BatchMetricCallback
classes: List[str] = None,
eps: float = 1e-7,
threshold: float = None,
activation: str = "Sigmoid",
) -> torch.Tensor:
"""
Args:
outputs (torch.Tensor): A list of predicted elements
targets (torch.Tensor): A list of elements that are to be predicted
classes (List[str]): if classes are specified
we reduce across all dims except channels
eps (float): epsilon to avoid zero division
threshold (float): threshold for outputs binarization
activation (str): An torch.nn activation applied to the outputs.
Must be one of ["none", "Sigmoid", "Softmax2d"]
Returns:
Union[float, List[float]]: IoU (Jaccard) score(s)
"""
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
if threshold is not None:
outputs = (outputs > threshold).float()
# TODO: fix classes issue
# ! fix backward compatibility
if classes is not None:
# if classes are specified we reduce across all dims except channels
sum_strategy = partial(torch.sum, dim=[0, 2, 3])
else:
sum_strategy = torch.sum
intersection = sum_strategy(targets * outputs)
union = sum_strategy(targets) + sum_strategy(outputs)
# this looks a bit awkward but `eps * (union == 0)` term
# makes sure that if I and U are both 0, than IoU == 1
# and if U != 0 and I == 0 the eps term in numerator is zeroed out
# i.e. (0 + eps) / (U - 0 + eps) doesn't happen
output_iou = (intersection + eps * (union == 0)) / (
union - intersection + eps
)
return output_iou
def dice(
outputs: T,
targets: T,
eps: float = 1e-7,
threshold: float = None,
activation: str = "Sigmoid",
) -> T:
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
if threshold is not None:
outputs = (outputs > threshold).float()
intersection = torch.sum(targets * outputs)
union = torch.sum(targets) + torch.sum(outputs)
score = (2 * intersection + eps * (union == 0)) / (union + eps)
return score
def f1_score(
outputs: torch.Tensor,
targets: torch.Tensor,
beta: float = 1.0,
eps: float = 1e-7,
threshold: float = None,
activation: str = "Sigmoid",
):
"""
Args:
outputs: A list of predicted elements
targets: A list of elements that are to be predicted
eps: epsilon to avoid zero division
beta: beta param for f_score
threshold: threshold for outputs binarization
activation: An torch.nn activation applied to the outputs.
Must be one of ["none", "Sigmoid", "Softmax2d"]
Returns:
float: F_1 score
"""
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
if threshold is not None:
outputs = (outputs > threshold).float()
true_positive = torch.sum(targets * outputs)
false_positive = torch.sum(outputs) - true_positive
false_negative = torch.sum(targets) - true_positive
precision_plus_recall = (
(1 + beta ** 2) * true_positive
+ beta ** 2 * false_negative
+ false_positive
+ eps
)
score = ((1 + beta ** 2) * true_positive + eps) / precision_plus_recall
return score
def accuracy(
outputs: torch.Tensor,
targets: torch.Tensor,
topk: Sequence[int] = (1, ),
activation: Optional[str] = None,
) -> Sequence[torch.Tensor]:
"""
Computes multi-class accuracy@topk for the specified values of `topk`.
Args:
outputs: model outputs, logits
with shape [bs; num_classes]
targets: ground truth, labels
with shape [bs; 1]
activation: activation to use for model output
topk: `topk` for accuracy@topk computing
Returns:
list with computed accuracy@topk
"""
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
max_k = max(topk)
batch_size = targets.size(0)
if len(outputs.shape) == 1 or outputs.shape[1] == 1:
# binary accuracy
pred = outputs.t()
else:
# multi-class accuracy
_, pred = outputs.topk(max_k, 1, True, True) # noqa: WPS425
pred = pred.t()
correct = pred.eq(targets.long().view(1, -1).expand_as(pred))
output = []
for k in topk:
correct_k = (correct[:k].contiguous().view(-1).float().sum(
0, keepdim=True))
output.append(correct_k.mul_(1.0 / batch_size))
return output
def tversky(
outputs: T,
targets: T,
alpha: float = 0.7,
eps: float = 1e-7,
threshold: float = None,
activation: str = "Sigmoid",
) -> T:
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
if threshold is not None:
outputs = (outputs > threshold).float()
tp = torch.sum(targets * outputs)
fn = torch.sum(targets * (1 - outputs))
fp = torch.sum((1 - targets) * outputs)
dn = tp + alpha * fn + (1 - alpha) * fp
score = (tp + eps * (dn == 0)) / (dn + eps)
return score
def __init__(
self,
metric_names: List[str],
meter_list: List,
input_key: str = "targets",
output_key: str = "logits",
class_names: List[str] = None,
num_classes: int = 2,
activation: str = "Sigmoid",
):
"""
Args:
metric_names: of metrics to print
Make sure that they are in the same order that metrics
are outputted by the meters in `meter_list`
meter_list: List of meters.meter.Meter instances
len(meter_list) == num_classes
input_key: input key to use for metric calculation
specifies our ``y_true``.
output_key: output key to use for metric calculation;
specifies our ``y_pred``
class_names: class names to display in the logs.
If None, defaults to indices for each class, starting from 0.
num_classes: Number of classes; must be > 1
activation: An torch.nn activation applied to the logits.
Must be one of ['none', 'Sigmoid', 'Softmax2d']
"""
super().__init__(CallbackOrder.metric)
self.metric_names = metric_names
self.meters = meter_list
self.input_key = input_key
self.output_key = output_key
self.class_names = class_names
self.num_classes = num_classes
self.activation = activation
self.activation_fn = get_activation_fn(self.activation)
def accuracy(
outputs: torch.Tensor,
targets: torch.Tensor,
topk: Tuple = (1, ),
threshold: float = None,
activation: str = None,
):
"""
Computes the accuracy.
It can be used either for:
1. Multi-class task, in this case:
- you can use topk.
- threshold and activation are not required.
- targets is a tensor: batch_size
- outputs is a tensor: batch_size x num_classes
- computes the accuracy@k for the specified values of k.
2. Multi-label task, in this case:
- you must specify threshold and activation
- topk will not be used
(because of there is no method to apply top-k in
multi-label classification).
- outputs, targets are tensors with shape: batch_size x num_classes
- targets is a tensor with binary vectors
Args:
outputs (torch.Tensor): model outputs, logits
targets (torch.Tensor): ground truth, labels
topk (tuple): tuple with specified `N` for top`N` accuracy computing
threshold (float): threshold for outputs
activation (str): activation for outputs
Returns:
computed topK accuracy
"""
activation_fn = get_activation_fn(activation)
outputs = activation_fn(outputs)
if threshold:
outputs = (outputs > threshold).long()
# TODO: move to separate function
# multi-label classification
if len(targets.shape) > 1 and targets.size(1) > 1:
output = (targets.long() == outputs.long()).sum().float() / np.prod(
targets.shape)
return [output]
max_k = max(topk)
batch_size = targets.size(0)
if len(outputs.shape) == 1 or outputs.shape[1] == 1:
pred = outputs.t()
else:
_, pred = outputs.topk(max_k, 1, True, True) # noqa: WPS425
pred = pred.t()
correct = pred.eq(targets.long().view(1, -1).expand_as(pred))
output = []
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
output.append(correct_k.mul_(1.0 / batch_size))
return output
