def compute_epoch_metrics(self, predictions: Any, targets: Any, classes: List[str] = None) -> dict: """Computes metrics for the epoch :param targets: ground truth :type targets: Any :param predictions: model predictions :type predictions: Any :param classes: list of classes in the target :type classes: List[str], defaults to None :return: dictionary of metrics as provided in the config file """ targets = targets.cpu() predicted_labels = torch.argmax(predictions, dim=1).detach().cpu() if classes is None: classes = self.model_config['classes'] confusion_matrix = ConfusionMatrix(classes) confusion_matrix(targets, predicted_labels) metrics = { 'accuracy': accuracy_score(targets, predicted_labels), 'confusion_matrix': confusion_matrix.cm, } return metrics
def test_tp_multi_class(self): """Checks calling tp on multi-class problem""" cm = ConfusionMatrix([0, 1, 2]) y_true = torch.tensor([0, 1, 2]) y_pred = torch.tensor([0, 1, 2]) cm(y_true, y_pred) with self.assertRaises(ValueError): cm.tp
def compute_epoch_metrics(self, predictions: Any, targets: Any, threshold: float = None, recall: float = 0.9, as_logits: bool = True, classes: List[str] = None) -> dict: """Computes metrics for the epoch :param targets: ground truth :type targets: Any :param predictions: model predictions :type predictions: Any :param threshold: confidence threshold to be used for binary classification; if None, the optimal threshold is found. :type threshold: float, defaults to None :param recall: minimum recall to choose the optimal threshold :type recall: float, defaults to 0.9 :param as_logits: whether the predictions are logits; if as_logits=True, the values are converted into sigmoid scores before further processing. :type as_logits: bool, defaults to True :param classes: list of classes in the target :type classes: List[str], defaults to None :return: dictionary of metrics as provided in the config file """ if as_logits: # convert to sigmoid scores from logits predictions = torch.sigmoid(predictions) targets = targets.cpu() predict_proba = predictions.detach().cpu() if classes is None: classes = self.model_config['classes'] if len(classes) != 2: raise ValueError('More than 2 classes found') if threshold is None: logging.info('Finding optimal threshold based on: {}'.format( self.model_config['eval']['maximize_metric'])) maximize_fn = metric_factory.create( self.model_config['eval']['maximize_metric'], **{'recall': recall}) _, _, threshold = maximize_fn(targets, predict_proba) predicted_labels = torch.ge(predict_proba, threshold).cpu() confusion_matrix = ConfusionMatrix(classes) confusion_matrix(targets, predicted_labels) tp = confusion_matrix.tp fp = confusion_matrix.fp tn = confusion_matrix.tn fp = confusion_matrix.fp metrics = { 'accuracy': accuracy_score(targets, predicted_labels), 'confusion_matrix': confusion_matrix.cm, 'precision': precision_score(targets, predicted_labels), 'recall': recall_score(targets, predicted_labels, zero_division=1), 'threshold': float(threshold), 'specificity': confusion_matrix.specificity } precisions, recalls, thresholds = precision_recall_curve( targets, predict_proba) metrics['pr-curve'] = plot_classification_metric_curve( recalls, precisions, xlabel='Recall', ylabel='Precision') plt.close() fprs, tprs, _ = roc_curve(targets, predict_proba) metrics['roc-curve'] = plot_classification_metric_curve( fprs, tprs, xlabel='False Positive Rate', ylabel='True Positive Rate') plt.close() if len(torch.unique(targets)) == 1: metrics['auc-roc'] = 0 else: metrics['auc-roc'] = roc_auc_score(targets, predict_proba) specificities = np.array([1 - fpr for fpr in fprs]) metrics['ss-curve'] = plot_classification_metric_curve( tprs, specificities, xlabel='Sensitivity', ylabel='Specificity') plt.close() return metrics
def test_tp(self): """Checks TP in the usual case""" cm = ConfusionMatrix([0, 1]) cm(self.y_true, self.y_pred) self.assertEqual(cm.tp, 2)
def test_confusion_matrix(self): """Checks the usual case""" cm = ConfusionMatrix([0, 1]) cm(self.y_true, self.y_pred) assert_array_equal(cm.cm, [[3, 2], [3, 2]])
def test_tp_before_cm(self): """Checks calling tp before computing confusion matrix""" cm = ConfusionMatrix([0, 1]) with self.assertRaises(ValueError): cm.tp
def test_specificity(self): """Checks specificity in the usual case""" cm = ConfusionMatrix([0, 1]) cm(self.y_true, self.y_pred) self.assertEqual(cm.specificity, 0.6)
def test_fn(self): """Checks FN in the usual case""" cm = ConfusionMatrix([0, 1]) cm(self.y_true, self.y_pred) self.assertEqual(cm.fn, 3)