def test_on_2_pixels_2mm_away(self): mask_gt = np.zeros((128, 128), np.bool) mask_pred = np.zeros((128, 128), np.bool) mask_gt[50, 70] = 1 mask_pred[50, 72] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(2, 1)) diag = 0.5 * math.sqrt(2**2 + 1**2) expected_distances = { 'surfel_areas_gt': np.asarray([diag, diag, diag, diag]), 'surfel_areas_pred': np.asarray([diag, diag, diag, diag]), 'distances_gt_to_pred': np.asarray([1., 1., 2., 2.]), 'distances_pred_to_gt': np.asarray([1., 1., 2., 2.]), } self.assertEqual(len(expected_distances), len(surface_distances)) for key, expected_value in expected_distances.items(): np.testing.assert_array_equal(expected_value, surface_distances[key]) self._assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(1.5, 1.5), expected_hausdorff_100=2.0, expected_hausdorff_95=2.0, expected_surface_overlap_at_1mm=(0.5, 0.5), expected_surface_dice_at_1mm=0.5, expected_volumetric_dice=0.0)
def testEmptyBothMasks(): mask_gt = np.zeros((128, 128, 128), np.uint8) mask_pred = np.zeros((128, 128, 128), np.uint8) surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(3, 2, 1)) assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(np.nan, np.nan), expected_hausdorff_100=np.inf, expected_hausdorff_95=np.inf, expected_surface_overlap_at_1mm=(np.nan, np.nan), expected_surface_dice_at_1mm=np.nan, expected_volumetric_dice=np.nan)
def test_both_empty_masks(self): mask_gt = np.zeros((128, 128, 128), np.bool) mask_pred = np.zeros((128, 128, 128), np.bool) surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(3, 2, 1)) self._assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(np.nan, np.nan), expected_hausdorff_100=np.inf, expected_hausdorff_95=np.inf, expected_surface_overlap_at_1mm=(np.nan, np.nan), expected_surface_dice_at_1mm=np.nan, expected_volumetric_dice=np.nan)
def testEmptyGroundTruthMask(): mask_gt = np.zeros((128, 128, 128), np.uint8) mask_pred = np.zeros((128, 128, 128), np.uint8) mask_pred[50, 60, 72] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(3, 2, 1)) assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(np.nan, np.inf), expected_hausdorff_100=np.inf, expected_hausdorff_95=np.inf, expected_surface_overlap_at_1mm=(np.nan, 0.0), expected_surface_dice_at_1mm=0.0, expected_volumetric_dice=0.0)
def test_two_cubes_shifted_by_one_pixel(self): mask_gt = np.zeros((100, 100, 100), np.bool) mask_pred = np.zeros((100, 100, 100), np.bool) mask_gt[0:50, :, :] = 1 mask_pred[0:51, :, :] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(2, 1, 1)) self._assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(0.322, 0.339), expected_hausdorff_100=2.0, expected_hausdorff_95=2.0, expected_surface_overlap_at_1mm=(0.842, 0.830), expected_surface_dice_at_1mm=0.836, expected_volumetric_dice=0.990)
def test_empty_ground_truth_mask(self): mask_gt = np.zeros((128, 128, 128), np.bool) mask_pred = np.zeros((128, 128, 128), np.bool) mask_pred[50, 60, 72] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(3, 2, 1)) self._assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(np.nan, np.inf), expected_hausdorff_100=np.inf, expected_hausdorff_95=np.inf, expected_surface_overlap_at_1mm=(np.nan, 0.0), expected_surface_dice_at_1mm=0.0, expected_volumetric_dice=0.0)
def testEmptyPredictionMask(self): mask_gt = np.zeros((128, 128, 128), np.uint8) mask_pred = np.zeros((128, 128, 128), np.uint8) mask_gt[50, 60, 70] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(3, 2, 1)) self._assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(np.inf, np.nan), expected_hausdorff_100=np.inf, expected_hausdorff_95=np.inf, expected_surface_overlap_at_1mm=(0.0, np.nan), expected_surface_dice_at_1mm=0.0, expected_volumetric_dice=0.0)
def testTwoCubes(): mask_gt = np.zeros((100, 100, 100), np.uint8) mask_pred = np.zeros((100, 100, 100), np.uint8) mask_gt[0:50, :, :] = 1 mask_pred[0:51, :, :] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(2, 1, 1)) assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(0.322, 0.339), expected_hausdorff_100=2.0, expected_hausdorff_95=2.0, expected_surface_overlap_at_1mm=(0.842, 0.830), expected_surface_dice_at_1mm=0.836, expected_volumetric_dice=0.990)
def testSinglePixels2mmAway(): mask_gt = np.zeros((128, 128, 128), np.uint8) mask_pred = np.zeros((128, 128, 128), np.uint8) mask_gt[50, 60, 70] = 1 mask_pred[50, 60, 72] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(3, 2, 1)) assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(1.5, 1.5), expected_hausdorff_100=2.0, expected_hausdorff_95=2.0, expected_surface_overlap_at_1mm=(0.5, 0.5), expected_surface_dice_at_1mm=0.5, expected_volumetric_dice=0.0)
def surface_distances(img_expert, img_model): surface_distances = surface_distance.compute_surface_distances( img_expert, img_model, (0.1, 0.1)) robust_hausdorff = surface_distance.compute_robust_hausdorff( surface_distances, 95) dist, dist_inv = surface_distance.compute_average_surface_distance( surface_distances) dice_at_tolerance = surface_distance.compute_surface_dice_at_tolerance( surface_distances, tolerance_mm=1.0) tmp = { "dist": dist, "dist_inv": dist_inv, "robust_hausdorff": robust_hausdorff, "dice_at_tolerance": dice_at_tolerance, } return tmp
def test_on_2_pixels_2mm_away(self): mask_gt = np.zeros((128, 128, 128), np.bool) mask_pred = np.zeros((128, 128, 128), np.bool) mask_gt[50, 60, 70] = 1 mask_pred[50, 60, 72] = 1 surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(3, 2, 1)) self._assert_metrics(surface_distances, mask_gt, mask_pred, expected_average_surface_distance=(1.5, 1.5), expected_hausdorff_100=2.0, expected_hausdorff_95=2.0, expected_surface_overlap_at_1mm=(0.5, 0.5), expected_surface_dice_at_1mm=0.5, expected_volumetric_dice=0.0)
def compute_metrics(mask_gt, mask_pred, spacing_mm=(3, 0.6, 0.6), surface_tolerances=None): if surface_tolerances is None: surface_tolerances = [1, 3] surface_DSC = [] DSC = [] HD_95 = [] surface_distances = surface_distance.compute_surface_distances(mask_gt.astype(int), mask_pred.astype(int), spacing_mm=spacing_mm) DSC.append(surface_distance.compute_dice_coefficient(mask_gt.astype(int), mask_pred.astype(int))) for surface_tolerance in surface_tolerances: surface_DSC.append(surface_distance.compute_surface_dice_at_tolerance(surface_distances, surface_tolerance)) HD_95.append(surface_distance.compute_robust_hausdorff(surface_distances, 95)) return DSC, surface_DSC, HD_95
def HD_3d(mask_path, pred_path, lobe_index): mask_sitk_img = sitk.ReadImage(mask_path) mask_img_arr = sitk.GetArrayFromImage(mask_sitk_img) pred_sitk_img = sitk.ReadImage(pred_path) pred_img_arr = sitk.GetArrayFromImage(pred_sitk_img) mask_img_arr[mask_img_arr != lobe_index] = 0 pred_img_arr[pred_img_arr != lobe_index] = 0 mask_img_arr[mask_img_arr == lobe_index] = 1 pred_img_arr[pred_img_arr == lobe_index] = 1 mask_img_arr = mask_img_arr.astype(np.bool_) pred_img_arr = pred_img_arr.astype(np.bool_) spacing = mask_sitk_img.GetSpacing() surface_distances = surfdist.compute_surface_distances(mask_img_arr, pred_img_arr, spacing_mm=spacing) hd_dist_95 = surfdist.compute_robust_hausdorff(surface_distances, 95) return hd_dist_95
def ASSD_3d(mask_path, pred_path, lobe_index): mask_sitk_img = sitk.ReadImage(mask_path) mask_img_arr = sitk.GetArrayFromImage(mask_sitk_img) pred_sitk_img = sitk.ReadImage(pred_path) pred_img_arr = sitk.GetArrayFromImage(pred_sitk_img) mask_img_arr[mask_img_arr != lobe_index] = 0 pred_img_arr[pred_img_arr != lobe_index] = 0 mask_img_arr[mask_img_arr == lobe_index] = 1 pred_img_arr[pred_img_arr == lobe_index] = 1 mask_img_arr = mask_img_arr.astype(np.bool_) pred_img_arr = pred_img_arr.astype(np.bool_) spacing = mask_sitk_img.GetSpacing() surface_distances = surfdist.compute_surface_distances(mask_img_arr, pred_img_arr, spacing_mm=spacing) asd = surfdist.compute_average_surface_distance(surface_distances) return np.mean(asd)
def eval_net(net_spinal_cord, net_gm, loader, writer, logger, threshold=0.5, epoch=None): """Evaluation without the densecrf with the dice coefficient""" net_gm.eval() net_spinal_cord.eval() resolution = loader.dataset.info_dict['resolution'] criterion = nn.BCEWithLogitsLoss() metric_dict = { 'DSC': metrics.DiceSimilarityCoefficient, 'JI': metrics.JaccardIndex, 'CC': metrics.ConformityCoefficient, 'TPR': metrics.Sensitivity, 'TNR': metrics.Specificity, 'PPV': metrics.Precision } transform_eval = T.ComposedTransform([T.CenterCrop(144)]) eval_result = {key: 0 for key in metric_dict.keys()} eval_result['loss_gm'] = 0 eval_result['loss_spinal_cord'] = 0 eval_result['auc'] = 0 eval_result['avg_surface_distance'] = 0 with torch.no_grad(): for data3D, gt_list in loader: gt_list = torch.cat([gt.byte() for gt in gt_list], 0) x, gt = data3D.transpose(0, 1), gt_list.transpose(0, 1) x_max = x.max(dim=2)[0].max(dim=2)[0] x_max = (x_max + (x_max == 0).float()).view(-1, 1, 1, 1) x = x / x_max true_masks = gt.cuda() spinal_cord_mask = (torch.mean( (true_masks > 0).float(), dim=1) > 0.5).unsqueeze( dim=1).float() gm_gt_mask = (torch.mean( (true_masks == 1).float(), dim=1) > 0.5).unsqueeze( dim=1).float() trans_x, trans_cord_mask, trans_gm_mask = [], [], [] for i in range(x.shape[0]): a, b, c = transform_eval(x[i], spinal_cord_mask[i], gm_gt_mask[i]) trans_x.append(a), trans_cord_mask.append( b), trans_gm_mask.append(c) x, spinal_cord_mask, gm_gt_mask = torch.stack( trans_x, dim=0).cuda(), torch.stack( trans_cord_mask, dim=0).cuda(), torch.stack(trans_gm_mask, dim=0).cuda() spinal_cord_pred, _ = net_spinal_cord(x) loss_spinal_cord = criterion(spinal_cord_pred, spinal_cord_mask) spinal_mask_pred = (torch.sigmoid(spinal_cord_pred) > 0.5).detach().float() local_max = (spinal_mask_pred * x).max(dim=2)[0].max(dim=2)[0] local_min = ((1 - spinal_mask_pred) * 9999 + spinal_mask_pred * x).min(dim=2)[0].min(dim=2)[0] local_max = local_max.view(-1, 1, 1, 1) local_min = local_min.view(-1, 1, 1, 1) local_min *= (local_min < 9000).float() x = torch.clamp( (x - local_min) / ((local_max - local_min) + ((local_max - local_min) == 0).float()), 0, 1) gm_pred, _ = net_gm(x) # * spinal_mask_pred gm_pos_weight = torch.sum(spinal_cord_mask) / torch.sum( spinal_cord_mask * gm_gt_mask) if torch.isinf(gm_pos_weight) or torch.isnan(gm_pos_weight): gm_pos_weight = torch.tensor(1.).cuda() loss_gm = F.binary_cross_entropy_with_logits( gm_pred * spinal_mask_pred, gm_gt_mask, pos_weight=gm_pos_weight) gm_pred_mask = torch.sigmoid(gm_pred * spinal_mask_pred) > 0.5 eval_result['loss_gm'] += loss_gm.item() eval_result['loss_spinal_cord'] += loss_spinal_cord.item() eval_result['auc'] += skmetrics.roc_auc_score( gm_gt_mask.view(-1).cpu().numpy(), gm_pred.view(-1).cpu().numpy()) surface_dis = surface_distance.compute_surface_distances( gm_gt_mask.cpu().squeeze().numpy().astype(np.bool), gm_pred_mask.cpu().squeeze().numpy().astype(np.bool), spacing_mm=resolution) eval_result['avg_surface_distance'] += np.mean( surface_distance.compute_average_surface_distance(surface_dis)) # gm_pred_full_size = (torch.sigmoid(gm_pred) * spinal_mask_pred) > 0.5 # writer.add_images('raw_data', x, global_step=epoch) # writer.add_images('SpinalList_Pred', spinal_mask_pred, global_step=epoch) # writer.add_images('SpinalList_GT', spinal_cord_mask, global_step=epoch) # writer.add_images('GmMask_Pred', gm_pred_mask, global_step=epoch) # writer.add_images('GmMask_GT', gm_gt_mask, global_step=epoch) for key, metric_func in metric_dict.items(): eval_result[key] += metric_func(gm_pred_mask, gm_gt_mask) for key, val in eval_result.items(): eval_result[key] = val / len(loader) # writer.add_scalar('eval/' + key, eval_result[key], global_step=epoch) info_list = [{ 'name': key, 'val': value } for key, value in eval_result.items()] logger.log_epoch_info(info_list, epoch=epoch) return eval_result
jc = intersection.sum() / float(union.sum()) jc = round(jc, 3) jc_i_train.append(jc) #HAUSDORFF DISTANCE #gdth_files = (gdth_path + f) for g in gdth_images_train: sx, sy, sz = gdth_train_data.header.get_zooms() #CONFIRMAR E FAZER COM QUE ESTEJA A IR BUSCAR O HEADER INFO DAS MESMAS IMAGENS QUE ESTAMOS A PROCESSAR DE CADA VEZ surface_distances = compute_surface_distances( gdth_train_arr_list[m], train_arr_l[n], (sx, sy, sz)) hd = compute_robust_hausdorff(surface_distances, percent=100) hd = round(hd, 3) hausdorff_train.append(hd) #AVERAGE DISTANCE #ad = compute_average_surface_distance(surface_distances) #average_mas.append(ad) #print("For the arr_gdth", m, gdth_arr_list[m].shape, "and the arr_pred", n, pred_arr_list[n].shape) #print("The dice score is:", dice_score) #print("The jaccard index is:", jc)
Estimated_lung.shape[0] * Estimated_lung.shape[1] * Estimated_lung.shape[2], 1) print(y_scores.shape) y_true = te_mask.reshape( te_mask.shape[0] * te_mask.shape[1] * te_mask.shape[2], 1) y_pred = np.where(y_scores > 0.8, 1, 0) y_true = np.where(y_true > 0.5, 1, 0) # Surface Distance Metrics: hausdorff_list = np.array([0 for i in range(predictions.shape[0])]) avg_surf_dist = np.array( [np.array([0, 0]) for i in range(predictions.shape[0])]) for i in range(len(predictions)): surface_dist_dict = sd.compute_surface_distances( predictions[i].astype(bool), Estimated_lung[i].astype(bool), [1, 1]) hausdorff_list[i] = sd.compute_robust_hausdorff(surface_dist_dict, 1) tmp_avg = sd.compute_average_surface_distance(surface_dist_dict) avg_surf_dist[i, 0] = tmp_avg[0] avg_surf_dist[i, 1] = tmp_avg[1] print("\nMean of Hausdorff distances: " + str(np.mean(hausdorff_list))) print("\nMean of Average Surface Distances (Original to predicted): " + str(np.mean(avg_surf_dist[:, 0]))) print("\nMean of Average Surface Distances (Predicted to original): " + str(np.mean(avg_surf_dist[:, 1]))) # Area under the ROC curve fpr, tpr, thresholds = roc_curve((y_true), y_pred) AUC_ROC = roc_auc_score(y_true, y_pred) print("\nArea under the ROC curve: " + str(AUC_ROC))
def test_compute_surface_distances_raises_on_incompatible_shapes( self, mask_gt, mask_pred, spacing_mm): with self.assertRaisesRegex( ValueError, 'The arguments must be of compatible shape'): surface_distance.compute_surface_distances(mask_gt, mask_pred, spacing_mm)
mask_pred_all = [] for batch in test_loader: imgs, true_masks = batch['image'], batch['mask'] imgs = imgs.to(device=device, dtype=torch.float32) true_masks = true_masks.cpu().numpy()[0, 0] true_masks_all.append(true_masks) with torch.no_grad(): mask_pred = net(imgs) mask_pred = torch.sigmoid(mask_pred) mask_pred = mask_pred.cpu().numpy()[0, 0] mask_pred = (mask_pred > 0.5) mask_pred_all.append(mask_pred) val_score = surface_distance.compute_surface_distances( np.array(mask_pred_all), np.array(true_masks_all, dtype=bool), [1, 1, 1]) dice = surface_distance.compute_surface_dice_at_tolerance( val_score, 1) print(str(name) + ': ' + "{:.4f}".format(dice)) """ Surface distance: 2mm philips_15 2D ge_15: 0.7882200408190674 ge_3: 0.6297838360700679 philips_15: 0.87457095303606 philips_3: 0.2743725966714217 siemens_15: 0.7327510562328482 siemens_3: 0.8081202860991571 """
def test_compute_surface_distances_raises_on_invalid_shapes( self, mask_gt, mask_pred, spacing_mm): with self.assertRaisesRegex(ValueError, 'Only 2D and 3D masks are supported'): surface_distance.compute_surface_distances(mask_gt, mask_pred, spacing_mm)
def surface_dice(x, y, spacing, tolerance=1.0): surf_dist = sd.compute_surface_distances(x, y, spacing) return sd.compute_surface_dice_at_tolerance(surf_dist, tolerance)
def test_two_squares_shifted_by_one_pixel(self): # We make sure we do not have active pixels on the border of the image, # because this will add additional 2D surfaces on the border of the image # because the image is padded with background. mask_gt = np.asarray([ [0, 0, 0, 0, 0, 0], [0, 1, 1, 0, 0, 0], [0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ], dtype=np.bool) mask_pred = np.asarray([ [0, 0, 0, 0, 0, 0], [0, 1, 1, 0, 0, 0], [0, 1, 1, 0, 0, 0], [0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ], dtype=np.bool) vertical = 2 horizontal = 1 diag = 0.5 * math.sqrt(horizontal**2 + vertical**2) surface_distances = surface_distance.compute_surface_distances( mask_gt, mask_pred, spacing_mm=(vertical, horizontal)) # We go from top left corner, clockwise to describe the surfaces and # distances. The 2 surfaces are: # # /-\ /-\ # | | | | # \-/ | | # \-/ expected_surfel_areas_gt = np.asarray([ diag, horizontal, diag, vertical, diag, horizontal, diag, vertical ]) expected_surfel_areas_pred = np.asarray([ diag, horizontal, diag, vertical, vertical, diag, horizontal, diag, vertical, vertical ]) expected_distances_gt_to_pred = np.asarray([0] * 5 + [horizontal] + [0] * 2) expected_distances_pred_to_gt = np.asarray([0] * 5 + [vertical] * 3 + [0] * 2) # We sort these using the same sorting algorithm (expected_distances_gt_to_pred, expected_surfel_areas_gt) = (metrics._sort_distances_surfels( expected_distances_gt_to_pred, expected_surfel_areas_gt)) (expected_distances_pred_to_gt, expected_surfel_areas_pred) = (metrics._sort_distances_surfels( expected_distances_pred_to_gt, expected_surfel_areas_pred)) expected_distances = { 'surfel_areas_gt': expected_surfel_areas_gt, 'surfel_areas_pred': expected_surfel_areas_pred, 'distances_gt_to_pred': expected_distances_gt_to_pred, 'distances_pred_to_gt': expected_distances_pred_to_gt, } self.assertEqual(len(expected_distances), len(surface_distances)) for key, expected_value in expected_distances.items(): np.testing.assert_array_equal(expected_value, surface_distances[key]) self._assert_metrics( surface_distances, mask_gt, mask_pred, expected_average_surface_distance=( surface_distance.compute_average_surface_distance( expected_distances)), expected_hausdorff_100=(surface_distance.compute_robust_hausdorff( expected_distances, 100)), expected_hausdorff_95=surface_distance.compute_robust_hausdorff( expected_distances, 95), expected_surface_overlap_at_1mm=( surface_distance.compute_surface_overlap_at_tolerance( expected_distances, tolerance_mm=1)), expected_surface_dice_at_1mm=( surface_distance.compute_surface_dice_at_tolerance( surface_distances, tolerance_mm=1)), expected_volumetric_dice=( surface_distance.compute_dice_coefficient(mask_gt, mask_pred)))
hd = 0 nh_batch = 0 for patient in patients: patientCRF = run_CRF(patient, sdimsB, sdimsG, schan).astype(np.uint64) patientGround = groundniiToNumpyCRFinput(patient, pathG).astype(np.uint64) roi = ROInii(patient, pathROI).astype(np.uint64) m1, m2, m3 = CRF_dice(patientCRF,patientGround,roi) print('patient:', patient) print ('dice-loss:', m1) print ('hard-dice:', m2) print ('iou:', m3) total_dice += m1 total_dice_hard += m2 total_iou += m3 surface_distances = sd.compute_surface_distances(patientCRF*roi, patientGround*roi, [1,1,1]) s3 = sd.compute_robust_hausdorff(surface_distances, 95) if s3 != np.Inf: hd += s3 print('patient:', patient, 'hausdorff distance:', s3) if s3 != np.Inf: nh_batch += 1 n_batch += 1 print ('avg-dice:', total_dice/n_batch) print ('avg-hard-dice:', total_dice_hard/n_batch) print ('avg-iou', total_iou/n_batch)