def create_generalized_dice_metric(self, cm: ConfusionMatrix, weight: torch.Tensor): """ Computes the Sørensen–Dice Coefficient (https://en.wikipedia.org/wiki/Sørensen–Dice_coefficient) Args: cm (:obj:`ignite.metrics.ConfusionMatrix`): A confusion matrix representing the classification of data. weight (:obj:`torch.Tensor`): A weight vector which length equals to the number of classes. Returns: ignite.Metric: The Generalized Dice Coefficient Metric object. """ # Increase floating point precision cm = cm.type(torch.float64) dice = 2 * (cm.diag() * weight) / (( (cm.sum(dim=1) + cm.sum(dim=0)) * weight) + EPSILON) if self._ignore_index != -100: def remove_index(dice_vector): try: indices = list(range(len(dice_vector))) indices.remove(self._ignore_index) return dice_vector[indices] except ValueError as e: raise IndexError( "'ignore_index' must be non-negative, and lower than the number of classes in confusion matrix, but {} was given. " .format(self._ignore_index)) return MetricsLambda(remove_index, dice) else: return dice
def save_confusion_matrix(data_root, output_root, segmenter, data_subset="val"): dataset = SegmentationDataset(data_root, data_subset) confusion_matrix_caluclator = ConfusionMatrix(num_classes=2, average="precision") accuracy_calculator = Accuracy() for image, mask_gt in dataset: mask_pred = segmenter.get_raw_prediction(image) mask_gt = torch.from_numpy(mask_gt).to( mask_pred.device).unsqueeze(0).unsqueeze(0) output = (mask_pred, mask_gt) confusion_matrix_caluclator.update( output_transform_confusion_matrix(output)) accuracy_calculator.update(output_transform_accuracy(output)) confusion_matrix = confusion_matrix_caluclator.compute() accuracy = accuracy_calculator.compute() cm_figure = plot_confusion_matrix(confusion_matrix) filename_base = f"confusion_matrix_acc={accuracy:.6f}" cm_figure.savefig(os.path.join(output_root, filename_base + ".pdf")) cm_figure.savefig(os.path.join(output_root, filename_base + ".png"))
def cmAccuracy(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: """Calculates accuracy using :class:`~ignite.metrics.ConfusionMatrix` metric. Args: cm (ConfusionMatrix): instance of confusion matrix metric Returns: MetricsLambda """ # Increase floating point precision and pass to CPU cm = cm.type(torch.DoubleTensor) correct_pixels = cm.diag() total_class_pixels = cm.sum(dim=1) pix_accs = correct_pixels / (total_class_pixels + 1e-15) if ignore_index is not None: def ignore_index_fn(pix_accs_vector): if ignore_index >= len(pix_accs_vector): raise ValueError( "ignore_index {} is larger than the length of pix_accs vector {}" .format(ignore_index, len(pix_accs_vector))) indices = list(range(len(pix_accs_vector))) indices.remove(ignore_index) return pix_accs_vector[indices] return MetricsLambda(ignore_index_fn, pix_accs) else: return pix_accs
class IoU(EvaluationMetric): """ Intersection over Union (IoU) metric per class The metric is defined for a pair of grayscale semantic segmentation images. Args: num_classes: number of calsses in the ground truth image output_transform: function that transform output pair of images Attributes: cm (ignite.metrics.ConfusionMatrix): pytorch ignite confusion matrix object. """ def __init__(self, num_classes, output_transform=lambda x: x): self.cm = ConfusionMatrix( num_classes=num_classes, average=None, output_transform=output_transform, ) def reset(self): self.cm.reset() def update(self, output): self.cm.update(output) def compute(self): cm = self.cm.compute() iou = cm.diag() / (cm.sum(dim=1) + cm.sum(dim=0) - cm.diag() + 1e-15) return iou
def __init__(self, num_classes: int, reduction: Union[None, str] = "mean", average: str = None, weight: torch.Tensor = None, ignore_index: int = -100, output_transform: callable = lambda x: x) -> None: """ Metric initializer. Args: num_classes (int): The number of classes in the problem. In case of images, num_classes should also count the background index 0. average (str, optional): Confusion matrix values averaging schema: None, "samples", "recall", "precision". Default is None. If `average="samples"` then confusion matrix values are normalized by the number of seen samples. If `average="recall"` then confusion matrix values are normalized such that diagonal values represent class recalls. If `average="precision"` then confusion matrix values are normalized such that diagonal values represent class precisions. reduction (str): The type of reduction to apply (e.g. 'mean'). ignore_index (int, optional): To ignore an index in Dice computation. output_transform (callable, optional): a callable that is used to transform the output into the form expected by the metric. This can be useful if, for example, you have a multi-output model and you want to compute the metric with respect to one of the outputs. """ if reduction not in self.SUPPORTED_REDUCTIONS: raise NotImplementedError("Reduction type not supported.") self._num_classes = num_classes self._ignore_index = ignore_index self._reduction = reduction self._weight = weight self._cm = ConfusionMatrix(num_classes=num_classes, average=average, output_transform=output_transform) self._metric = self.create_dice_metric(self._cm) super(Dice, self).__init__(output_transform=output_transform)
def test_no_update(): cm = ConfusionMatrix(10) with pytest.raises( NotComputableError, match=r"Confusion matrix must have at least one example before it " ): cm.compute()
def test_ignored_out_of_num_classes_indices(): num_classes = 21 cm = ConfusionMatrix(num_classes=num_classes) y_pred = torch.rand(4, num_classes, 12, 10) y = torch.randint(0, 255, size=(4, 12, 10)).long() cm.update((y_pred, y)) np_y_pred = y_pred.numpy().argmax(axis=1).ravel() np_y = y.numpy().ravel() assert np.all(confusion_matrix(np_y, np_y_pred, labels=list(range(num_classes))) == cm.compute().numpy())
def test_indexing_metric(): def _test(ignite_metric, sklearn_metic, sklearn_args, index, num_classes=5): y_pred = torch.rand(15, 10, num_classes).float() y = torch.randint(0, num_classes, size=(15, 10)).long() def update_fn(engine, batch): y_pred, y = batch return y_pred, y metrics = {'metric': ignite_metric[index], 'metric_wo_index': ignite_metric} validator = Engine(update_fn) for name, metric in metrics.items(): metric.attach(validator, name) def data(y_pred, y): for i in range(y_pred.shape[0]): yield (y_pred[i], y[i]) d = data(y_pred, y) state = validator.run(d, max_epochs=1) sklearn_output = sklearn_metic(y.view(-1).numpy(), y_pred.view(-1, num_classes).argmax(dim=1).numpy(), **sklearn_args) assert (state.metrics['metric_wo_index'][index] == state.metrics['metric']).all() assert (np.allclose(state.metrics['metric'].numpy(), sklearn_output)) num_classes = 5 labels = list(range(0, num_classes, 2)) _test(Precision(), precision_score, {'labels': labels, 'average': None}, index=labels) labels = list(range(num_classes - 1, 0, -2)) _test(Precision(), precision_score, {'labels': labels, 'average': None}, index=labels) labels = [1] _test(Precision(), precision_score, {'labels': labels, 'average': None}, index=labels) labels = list(range(0, num_classes, 2)) _test(Recall(), recall_score, {'labels': labels, 'average': None}, index=labels) labels = list(range(num_classes - 1, 0, -2)) _test(Recall(), recall_score, {'labels': labels, 'average': None}, index=labels) labels = [1] _test(Recall(), recall_score, {'labels': labels, 'average': None}, index=labels) # np.ix_ is used to allow for a 2D slice of a matrix. This is required to get accurate result from # ConfusionMatrix. ConfusionMatrix must be sliced the same row-wise and column-wise. labels = list(range(0, num_classes, 2)) _test(ConfusionMatrix(num_classes), confusion_matrix, {'labels': labels}, index=np.ix_(labels, labels)) labels = list(range(num_classes - 1, 0, -2)) _test(ConfusionMatrix(num_classes), confusion_matrix, {'labels': labels}, index=np.ix_(labels, labels)) labels = [1] _test(ConfusionMatrix(num_classes), confusion_matrix, {'labels': labels}, index=np.ix_(labels, labels))
def test_multiclass_images(): num_classes = 3 cm = ConfusionMatrix(num_classes=num_classes) y_true, y_pred = get_y_true_y_pred() # Compute confusion matrix with sklearn true_res = confusion_matrix(y_true.reshape(-1), y_pred.reshape(-1)) th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = cm.compute().numpy() assert np.all(true_res == res) # Another test on batch of 2 images num_classes = 3 cm = ConfusionMatrix(num_classes=num_classes) # Create a batch of two images: th_y_true1 = torch.from_numpy(y_true).reshape(1, 30, 30) th_y_true2 = torch.from_numpy(y_true.transpose()).reshape(1, 30, 30) th_y_true = torch.cat([th_y_true1, th_y_true2], dim=0) # Create a batch of 2 logits tensors y_probas = np.ones((3, 30, 30)) * -10 y_probas[0, (y_pred == 0)] = 720 y_probas[1, (y_pred == 1)] = 720 y_probas[2, (y_pred == 2)] = 768 th_y_logits1 = torch.from_numpy(y_probas).reshape(1, 3, 30, 30) y_probas = np.ones((3, 30, 30)) * -10 y_probas[0, (y_pred.transpose() == 0)] = 720 y_probas[1, (y_pred.transpose() == 2)] = 720 y_probas[2, (y_pred.transpose() == 1)] = 768 th_y_logits2 = torch.from_numpy(y_probas).reshape(1, 3, 30, 30) th_y_logits = torch.cat([th_y_logits1, th_y_logits2], dim=0) # Update metric & compute output = (th_y_logits, th_y_true) cm.update(output) res = cm.compute().numpy() # Compute confusion matrix with sklearn true_res = confusion_matrix( th_y_true.numpy().reshape(-1), np.argmax(th_y_logits.numpy(), axis=1).reshape(-1), ) assert np.all(true_res == res)
def test_cm_recall(): y_true, y_pred = np.random.randint(0, 10, size=(1000,)), np.random.randint(0, 10, size=(1000,)) th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) true_re = recall_score(y_true.reshape(-1), y_pred.reshape(-1), average="macro") cm = ConfusionMatrix(num_classes=10) re_metric = cmRecall(cm, average=True) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = re_metric.compute().numpy() assert pytest.approx(res) == true_re true_re = recall_score(y_true.reshape(-1), y_pred.reshape(-1), average=None) cm = ConfusionMatrix(num_classes=10) re_metric = cmRecall(cm, average=False) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = re_metric.compute().numpy() assert np.all(res == true_re)
def test_iou(): y_true, y_pred = get_y_true_y_pred() th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) true_res = [0, 0, 0] for index in range(3): bin_y_true = y_true == index bin_y_pred = y_pred == index intersection = bin_y_true & bin_y_pred union = bin_y_true | bin_y_pred true_res[index] = intersection.sum() / union.sum() cm = ConfusionMatrix(num_classes=3) iou_metric = IoU(cm) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = iou_metric.compute().numpy() assert np.all(res == true_res) for ignore_index in range(3): cm = ConfusionMatrix(num_classes=3) iou_metric = IoU(cm, ignore_index=ignore_index) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = iou_metric.compute().numpy() true_res_ = true_res[:ignore_index] + true_res[ignore_index + 1:] assert np.all(res == true_res_), "{}: {} vs {}".format( ignore_index, res, true_res_)
def test_dice_coefficient(): y_true, y_pred = get_y_true_y_pred() th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) true_res = [0, 0, 0] for index in range(3): bin_y_true = y_true == index bin_y_pred = y_pred == index # dice coefficient: 2*intersection(x, y) / (|x| + |y|) # union(x, y) = |x| + |y| - intersection(x, y) intersection = bin_y_true & bin_y_pred union = bin_y_true | bin_y_pred true_res[index] = 2.0 * intersection.sum() / (union.sum() + intersection.sum()) cm = ConfusionMatrix(num_classes=3) dice_metric = DiceCoefficient(cm) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = dice_metric.compute().numpy() np.testing.assert_allclose(res, true_res) for ignore_index in range(3): cm = ConfusionMatrix(num_classes=3) dice_metric = DiceCoefficient(cm, ignore_index=ignore_index) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = dice_metric.compute().numpy() true_res_ = true_res[:ignore_index] + true_res[ignore_index + 1 :] assert np.all(res == true_res_), f"{ignore_index}: {res} vs {true_res_}"
def _test(average=None): y_true, y_pred = get_y_true_y_pred() th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) true_res = [0, 0, 0] for index in range(3): bin_y_true = y_true == index bin_y_pred = y_pred == index intersection = bin_y_true & bin_y_pred union = bin_y_true | bin_y_pred true_res[index] = intersection.sum() / union.sum() cm = ConfusionMatrix(num_classes=3, average=average) jaccard_index = JaccardIndex(cm) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = jaccard_index.compute().numpy() assert np.all(res == true_res) for ignore_index in range(3): cm = ConfusionMatrix(num_classes=3) jaccard_index_metric = JaccardIndex(cm, ignore_index=ignore_index) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = jaccard_index_metric.compute().numpy() true_res_ = true_res[:ignore_index] + true_res[ignore_index + 1 :] assert np.all(res == true_res_), f"{ignore_index}: {res} vs {true_res_}"
def test_miou(): y_true, y_pred = get_y_true_y_pred() th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) true_res = [0, 0, 0] for index in range(3): bin_y_true = y_true == index bin_y_pred = y_pred == index intersection = bin_y_true & bin_y_pred union = bin_y_true | bin_y_pred true_res[index] = intersection.sum() / union.sum() true_res_ = np.mean(true_res) cm = ConfusionMatrix(num_classes=3) iou_metric = mIoU(cm) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = iou_metric.compute().numpy() assert res == true_res_ for ignore_index in range(3): cm = ConfusionMatrix(num_classes=3) iou_metric = mIoU(cm, ignore_index=ignore_index) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = iou_metric.compute().numpy() true_res_ = np.mean(true_res[:ignore_index] + true_res[ignore_index + 1 :]) assert res == true_res_, f"{ignore_index}: {res} vs {true_res_}"
def eval_model(model, val_loader, device='cpu', num_classes=21): def evaluate_function(engine, batch): model.eval() with torch.no_grad(): img, mask = batch img = img.to(device) mask = mask.to(device) mask_pred = model(img) try: mask_pred = mask_pred['out'] except: print('') return mask_pred, mask val_evaluator = Engine(evaluate_function) cm = ConfusionMatrix(num_classes=num_classes) mIoU(cm).attach(val_evaluator, 'mean IoU') Accuracy().attach(val_evaluator, "accuracy") Loss(loss_fn=nn.CrossEntropyLoss())\ .attach(val_evaluator, "CE Loss") state = val_evaluator.run(val_loader) #print("mIoU :",state.metrics['mean IoU']) #print("Accuracy :",state.metrics['accuracy']) #print("CE Loss :",state.metrics['CE Loss']) return state
def test_cm_accuracy(): y_true, y_pred = get_y_true_y_pred() th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) true_acc = accuracy_score(y_true.reshape(-1), y_pred.reshape(-1)) cm = ConfusionMatrix(num_classes=3) acc_metric = cmAccuracy(cm) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = acc_metric.compute().numpy() assert pytest.approx(res) == true_acc
def evaluation(local_rank, config, logger, with_clearml): rank = idist.get_rank() device = idist.device() manual_seed(config.seed + local_rank) data_loader = config.data_loader model = config.model.to(device) # Load weights: state_dict = get_model_weights(config, logger, with_clearml) model.load_state_dict(state_dict) # Adapt model to dist config model = idist.auto_model(model) # Setup evaluators num_classes = config.num_classes cm_metric = ConfusionMatrix(num_classes=num_classes) val_metrics = { "IoU": IoU(cm_metric), "mIoU_bg": mIoU(cm_metric), } if ("val_metrics" in config) and isinstance(config.val_metrics, dict): val_metrics.update(config.val_metrics) evaluator = create_evaluator(model, val_metrics, config, with_clearml, tag="val") # Setup Tensorboard logger if rank == 0: tb_logger = common.TensorboardLogger( log_dir=config.output_path.as_posix()) tb_logger.attach_output_handler( evaluator, event_name=Events.COMPLETED, tag="validation", metric_names="all", ) # Log confusion matrix to ClearML: if with_clearml: evaluator.add_event_handler(Events.COMPLETED, compute_and_log_cm, cm_metric, evaluator.state.iteration) state = evaluator.run(data_loader) utils.log_metrics(logger, 0, state.times["COMPLETED"], "Validation", state.metrics) if idist.get_rank() == 0: tb_logger.close()
def _test_distrib_accumulator_device(device): metric_devices = [torch.device("cpu")] if device.type != "xla": metric_devices.append(idist.device()) for metric_device in metric_devices: cm = ConfusionMatrix(num_classes=3, device=metric_device) assert cm._device == metric_device assert ( cm.confusion_matrix.device == metric_device ), f"{type(cm.confusion_matrix.device)}:{cm._num_correct.device} vs {type(metric_device)}:{metric_device}" y_true, y_pred = get_y_true_y_pred() th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) cm.update((th_y_logits, th_y_true)) assert ( cm.confusion_matrix.device == metric_device ), f"{type(cm.confusion_matrix.device)}:{cm._num_correct.device} vs {type(metric_device)}:{metric_device}"
def make_engine(process_function): evaluator = Engine(process_function) cm = ConfusionMatrix(num_classes=getattr( datasets, CONFIG["dataset"]["name"]).N_LABELS, output_transform=output_transform) IoU(cm, ignore_index=0).attach(evaluator, 'IoU') mIoU(cm, ignore_index=0).attach(evaluator, 'mIoU') Accuracy(output_transform=output_transform).attach(evaluator, 'Accuracy') cmAccuracy(cm, ignore_index=0).attach(evaluator, 'ClasswiseAccuracy') return evaluator
def get_metrics(loss): metrics = { "loss": Loss(loss), "accuracy": Accuracy(output_transform_accuracy), "confusion matrix": ConfusionMatrix(num_classes=2, output_transform=output_transform_confusion_matrix, average="precision"), } return metrics
def test_multiclass_input(): def _test(y_pred, y, num_classes, cm, batch_size): cm.reset() if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size cm.update( (y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) else: cm.update((y_pred, y)) np_y_pred = y_pred.numpy().argmax(axis=1).ravel() np_y = y.numpy().ravel() assert np.all( confusion_matrix(np_y, np_y_pred, labels=list(range(num_classes))) == cm.compute().numpy()) def get_test_cases(): return [ # Multiclass input data of shape (N, ) (torch.rand(10, 4), torch.randint(0, 4, size=(10, )).long(), 4, 1), (torch.rand(4, 10), torch.randint(0, 10, size=(4, )).long(), 10, 1), (torch.rand(4, 2), torch.randint(0, 2, size=(4, )).long(), 2, 1), (torch.rand(100, 5), torch.randint(0, 5, size=(100, )).long(), 5, 16), # Multiclass input data of shape (N, L) (torch.rand(10, 4, 5), torch.randint(0, 4, size=(10, 5)).long(), 4, 1), (torch.rand(4, 10, 5), torch.randint(0, 10, size=(4, 5)).long(), 10, 1), (torch.rand(100, 9, 7), torch.randint(0, 9, size=(100, 7)).long(), 9, 16), # Multiclass input data of shape (N, H, W, ...) (torch.rand(4, 5, 12, 10), torch.randint(0, 5, size=(4, 12, 10)).long(), 5, 1), (torch.rand(4, 5, 10, 12, 8), torch.randint(0, 5, size=(4, 10, 12, 8)).long(), 5, 1), (torch.rand(100, 3, 8, 8), torch.randint(0, 3, size=(100, 8, 8)).long(), 3, 16), ] # check multiple random inputs as random exact occurencies are rare for _ in range(5): for y_pred, y, num_classes, batch_size in get_test_cases(): cm = ConfusionMatrix(num_classes=num_classes) _test(y_pred, y, num_classes, cm, batch_size)
def create_dice_metric(self, cm: ConfusionMatrix): """ Computes the Sørensen–Dice Coefficient (https://en.wikipedia.org/wiki/Sørensen–Dice_coefficient) Args: cm (:obj:`ignite.metrics.ConfusionMatrix`): A confusion matrix representing the classification of data. Returns: array or float: The Sørensen–Dice Coefficient for each class or the mean Sørensen–Dice Coefficient. """ # Increase floating point precision cm = cm.type(torch.float64) dice = 2 * cm.diag() / (cm.sum(dim=1) + cm.sum(dim=0) + EPSILON) if self._ignore_index != -100: def remove_index(dice_vector): try: indices = list(range(len(dice_vector))) indices.remove(self._ignore_index) return dice_vector[indices] except ValueError as e: raise IndexError( "'ignore_index' must be non-negative, and lower than the number of classes in confusion matrix, but {} was given. " .format(self._ignore_index)) dice = MetricsLambda(remove_index, dice) if self._weight is not None: def multiply_weights(dice_vector): return self._weight * dice_vector dice = MetricsLambda(multiply_weights, dice) if self._reduction == "mean": dice = dice.mean() return dice
def test_network(model, test_dataset): with torch.no_grad(): evaluator = create_supervised_evaluator(model, metrics={ "accuracy": Accuracy(), "confusion": ConfusionMatrix(10) }) data_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False) evaluator.run(data_loader) return evaluator.state.metrics[ "accuracy"] * 100, evaluator.state.metrics["confusion"]
def __load_metrics(self): precision = Precision(average=False) recall = Recall(average=False) F1 = precision * recall * 2 / (precision + recall + 1e-20) F1 = MetricsLambda(lambda t: torch.mean(t).item(), F1) confusion_matrix = ConfusionMatrix(self.n_class, average="recall") # TODO: Add metric by patient self.metrics = { 'accuracy': Accuracy(), "f1": F1, "confusion_matrix": confusion_matrix, "precision": precision.mean(), "recall": recall.mean(), 'loss': Loss(self.loss) }
def evaluate(model, dataset, task_id, batch_size, device, out_id=0): val_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False) n_classes = dataset.tensors[1][:, 0].unique().numel() out_transform = get_attr_transform(out_id) eval_metrics = { 'accuracy': Accuracy(output_transform=out_transform), 'confusion': ConfusionMatrix(num_classes=n_classes, output_transform=out_transform) } evaluator = create_supervised_evaluator(model, metrics=eval_metrics, device=device) evaluator._logger.setLevel(logging.WARNING) evaluator.run(val_loader) metrics = evaluator.state.metrics return metrics['accuracy'], metrics['confusion']
def compute_and_log_cm(): cm = cm_metric.compute() # CM: values are normalized such that diagonal values represent class recalls cm = ConfusionMatrix.normalize(cm, "recall").cpu().numpy() if idist.get_rank() == 0: from trains import Task trains_logger = Task.current_task().get_logger() trains_logger.report_confusion_matrix( title="Final Confusion Matrix", series="cm-preds-gt", matrix=cm, iteration=trainer.state.iteration, xlabels=VOCSegmentationOpencv.target_names, ylabels=VOCSegmentationOpencv.target_names, )
def build_metrics(): metrics = { 'precision': Precision(lambda x: (x[0].argmax(dim=1), x[1])), 'recall': Recall(lambda x: (x[0].argmax(dim=1), x[1])), 'accuracy': Accuracy(lambda x: (x[0].argmax(dim=1), x[1])), 'confusion_matrix': ConfusionMatrix(2, output_transform=prepare_confusion_matrix), 'metric_output_results': MetricOutputResults(lambda x: (x[0].argmax(dim=1), x[1], x[0], x[2])), 'metric_last_layer': MetricLastLayer(lambda x: (x[3])), #'auroc':AUROC(lambda x: (x[0], x[1])), } return metrics
def test_dice_coefficient_wrong_input(): with pytest.raises(TypeError, match="Argument cm should be instance of ConfusionMatrix"): DiceCoefficient(None) cm = ConfusionMatrix(num_classes=10) with pytest.raises(ValueError, match="ignore_index should be non-negative integer"): DiceCoefficient(cm, ignore_index=-1) with pytest.raises(ValueError, match="ignore_index should be non-negative integer"): DiceCoefficient(cm, ignore_index="a") with pytest.raises(ValueError, match="ignore_index should be non-negative integer"): DiceCoefficient(cm, ignore_index=10) with pytest.raises(ValueError, match="ignore_index should be non-negative integer"): DiceCoefficient(cm, ignore_index=11)
def test_iou(): def _test(average=None): y_true, y_pred = get_y_true_y_pred() th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) true_res = [0, 0, 0] for index in range(3): bin_y_true = y_true == index bin_y_pred = y_pred == index intersection = bin_y_true & bin_y_pred union = bin_y_true | bin_y_pred true_res[index] = intersection.sum() / union.sum() cm = ConfusionMatrix(num_classes=3, average=average) iou_metric = IoU(cm) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = iou_metric.compute().numpy() assert np.all(res == true_res) for ignore_index in range(3): cm = ConfusionMatrix(num_classes=3) iou_metric = IoU(cm, ignore_index=ignore_index) # Update metric output = (th_y_logits, th_y_true) cm.update(output) res = iou_metric.compute().numpy() true_res_ = true_res[:ignore_index] + true_res[ignore_index + 1:] assert np.all(res == true_res_), "{}: {} vs {}".format( ignore_index, res, true_res_) _test() _test(average="samples") with pytest.raises( ValueError, match=r"ConfusionMatrix should have average attribute either"): cm = ConfusionMatrix(num_classes=3, average="precision") IoU(cm)
def step_train_supervised(model, train_loader, criterion, optimizer, device='cpu', num_classes=21): """ A step of fully supervised segmentation model training. """ def train_function(engine, batch): optimizer.zero_grad() model.train() img, mask = batch img = img.to(device) mask = mask.to(device) mask_pred = model(img) try: mask_pred = mask_pred['out'] except: print('') #print(mask_pred) #print("UNIQUE",torch.unique(mask_pred.argmax(dim=1))) #print("SIZE",mask_pred.size()) loss = criterion(mask_pred, mask) loss.backward() optimizer.step() return mask_pred, mask #print(num_classes) train_engine = Engine(train_function) cm = ConfusionMatrix( num_classes=num_classes) #,output_transform=output_transform) mIoU(cm).attach(train_engine, 'mean IoU') Accuracy().attach(train_engine, "accuracy") Loss(loss_fn=nn.CrossEntropyLoss()).attach(train_engine, "CE Loss") state = train_engine.run(train_loader) #print("mIoU :",state.metrics['mean IoU']) #print("Accuracy :",state.metrics['accuracy']) #print("CE Loss :",state.metrics['CE Loss']) return state