def compute_epoch_metrics(self,
predictions: Any,
targets: Any,
classes: List[str] = None) -> dict:
"""Computes metrics for the epoch
:param targets: ground truth
:type targets: Any
:param predictions: model predictions
:type predictions: Any
:param classes: list of classes in the target
:type classes: List[str], defaults to None
:return: dictionary of metrics as provided in the config file
"""
targets = targets.cpu()
predicted_labels = torch.argmax(predictions, dim=1).detach().cpu()
if classes is None:
classes = self.model_config['classes']
confusion_matrix = ConfusionMatrix(classes)
confusion_matrix(targets, predicted_labels)
metrics = {
'accuracy': accuracy_score(targets, predicted_labels),
'confusion_matrix': confusion_matrix.cm,
}
return metrics
def test_tp_multi_class(self):
"""Checks calling tp on multi-class problem"""
cm = ConfusionMatrix([0, 1, 2])
y_true = torch.tensor([0, 1, 2])
y_pred = torch.tensor([0, 1, 2])
cm(y_true, y_pred)
with self.assertRaises(ValueError):
cm.tp
def compute_epoch_metrics(self,
predictions: Any,
targets: Any,
threshold: float = None,
recall: float = 0.9,
as_logits: bool = True,
classes: List[str] = None) -> dict:
"""Computes metrics for the epoch
:param targets: ground truth
:type targets: Any
:param predictions: model predictions
:type predictions: Any
:param threshold: confidence threshold to be used for binary
classification; if None, the optimal threshold is found.
:type threshold: float, defaults to None
:param recall: minimum recall to choose the optimal threshold
:type recall: float, defaults to 0.9
:param as_logits: whether the predictions are logits; if
as_logits=True, the values are converted into sigmoid scores
before further processing.
:type as_logits: bool, defaults to True
:param classes: list of classes in the target
:type classes: List[str], defaults to None
:return: dictionary of metrics as provided in the config file
"""
if as_logits:
# convert to sigmoid scores from logits
predictions = torch.sigmoid(predictions)
targets = targets.cpu()
predict_proba = predictions.detach().cpu()
if classes is None:
classes = self.model_config['classes']
if len(classes) != 2:
raise ValueError('More than 2 classes found')
if threshold is None:
logging.info('Finding optimal threshold based on: {}'.format(
self.model_config['eval']['maximize_metric']))
maximize_fn = metric_factory.create(
self.model_config['eval']['maximize_metric'],
**{'recall': recall})
_, _, threshold = maximize_fn(targets, predict_proba)
predicted_labels = torch.ge(predict_proba, threshold).cpu()
confusion_matrix = ConfusionMatrix(classes)
confusion_matrix(targets, predicted_labels)
tp = confusion_matrix.tp
fp = confusion_matrix.fp
tn = confusion_matrix.tn
fp = confusion_matrix.fp
metrics = {
'accuracy': accuracy_score(targets, predicted_labels),
'confusion_matrix': confusion_matrix.cm,
'precision': precision_score(targets, predicted_labels),
'recall': recall_score(targets, predicted_labels, zero_division=1),
'threshold': float(threshold),
'specificity': confusion_matrix.specificity
}
precisions, recalls, thresholds = precision_recall_curve(
targets, predict_proba)
metrics['pr-curve'] = plot_classification_metric_curve(
recalls, precisions, xlabel='Recall', ylabel='Precision')
plt.close()
fprs, tprs, _ = roc_curve(targets, predict_proba)
metrics['roc-curve'] = plot_classification_metric_curve(
fprs,
tprs,
xlabel='False Positive Rate',
ylabel='True Positive Rate')
plt.close()
if len(torch.unique(targets)) == 1:
metrics['auc-roc'] = 0
else:
metrics['auc-roc'] = roc_auc_score(targets, predict_proba)
specificities = np.array([1 - fpr for fpr in fprs])
metrics['ss-curve'] = plot_classification_metric_curve(
tprs, specificities, xlabel='Sensitivity', ylabel='Specificity')
plt.close()
return metrics
def test_tp(self):
"""Checks TP in the usual case"""
cm = ConfusionMatrix([0, 1])
cm(self.y_true, self.y_pred)
self.assertEqual(cm.tp, 2)
def test_confusion_matrix(self):
"""Checks the usual case"""
cm = ConfusionMatrix([0, 1])
cm(self.y_true, self.y_pred)
assert_array_equal(cm.cm, [[3, 2], [3, 2]])
def test_tp_before_cm(self):
"""Checks calling tp before computing confusion matrix"""
cm = ConfusionMatrix([0, 1])
with self.assertRaises(ValueError):
cm.tp
def test_specificity(self):
"""Checks specificity in the usual case"""
cm = ConfusionMatrix([0, 1])
cm(self.y_true, self.y_pred)
self.assertEqual(cm.specificity, 0.6)
def test_fn(self):
"""Checks FN in the usual case"""
cm = ConfusionMatrix([0, 1])
cm(self.y_true, self.y_pred)
self.assertEqual(cm.fn, 3)