def surface_distances(img_expert, img_model):
surface_distances = surface_distance.compute_surface_distances(
img_expert, img_model, (0.1, 0.1))
robust_hausdorff = surface_distance.compute_robust_hausdorff(
surface_distances, 95)
dist, dist_inv = surface_distance.compute_average_surface_distance(
surface_distances)
dice_at_tolerance = surface_distance.compute_surface_dice_at_tolerance(
surface_distances, tolerance_mm=1.0)
tmp = {
"dist": dist,
"dist_inv": dist_inv,
"robust_hausdorff": robust_hausdorff,
"dice_at_tolerance": dice_at_tolerance,
}
return tmp
def _assert_metrics(self,
surface_distances,
mask_gt,
mask_pred,
expected_average_surface_distance,
expected_hausdorff_100,
expected_hausdorff_95,
expected_surface_overlap_at_1mm,
expected_surface_dice_at_1mm,
expected_volumetric_dice,
places=3):
actual_average_surface_distance = (
surface_distance.compute_average_surface_distance(
surface_distances))
for i in range(2):
self._assert_almost_equal(expected_average_surface_distance[i],
actual_average_surface_distance[i],
places=places)
self._assert_almost_equal(expected_hausdorff_100,
surface_distance.compute_robust_hausdorff(
surface_distances, 100),
places=places)
self._assert_almost_equal(expected_hausdorff_95,
surface_distance.compute_robust_hausdorff(
surface_distances, 95),
places=places)
actual_surface_overlap_at_1mm = (
surface_distance.compute_surface_overlap_at_tolerance(
surface_distances, tolerance_mm=1))
for i in range(2):
self._assert_almost_equal(expected_surface_overlap_at_1mm[i],
actual_surface_overlap_at_1mm[i],
places=places)
self._assert_almost_equal(
expected_surface_dice_at_1mm,
surface_distance.compute_surface_dice_at_tolerance(
surface_distances, tolerance_mm=1),
places=places)
self._assert_almost_equal(expected_volumetric_dice,
surface_distance.compute_dice_coefficient(
mask_gt, mask_pred),
places=places)
def ASSD_3d(mask_path, pred_path, lobe_index):
mask_sitk_img = sitk.ReadImage(mask_path)
mask_img_arr = sitk.GetArrayFromImage(mask_sitk_img)
pred_sitk_img = sitk.ReadImage(pred_path)
pred_img_arr = sitk.GetArrayFromImage(pred_sitk_img)
mask_img_arr[mask_img_arr != lobe_index] = 0
pred_img_arr[pred_img_arr != lobe_index] = 0
mask_img_arr[mask_img_arr == lobe_index] = 1
pred_img_arr[pred_img_arr == lobe_index] = 1
mask_img_arr = mask_img_arr.astype(np.bool_)
pred_img_arr = pred_img_arr.astype(np.bool_)
spacing = mask_sitk_img.GetSpacing()
surface_distances = surfdist.compute_surface_distances(mask_img_arr,
pred_img_arr,
spacing_mm=spacing)
asd = surfdist.compute_average_surface_distance(surface_distances)
return np.mean(asd)
def eval_net(net_spinal_cord,
net_gm,
loader,
writer,
logger,
threshold=0.5,
epoch=None):
"""Evaluation without the densecrf with the dice coefficient"""
net_gm.eval()
net_spinal_cord.eval()
resolution = loader.dataset.info_dict['resolution']
criterion = nn.BCEWithLogitsLoss()
metric_dict = {
'DSC': metrics.DiceSimilarityCoefficient,
'JI': metrics.JaccardIndex,
'CC': metrics.ConformityCoefficient,
'TPR': metrics.Sensitivity,
'TNR': metrics.Specificity,
'PPV': metrics.Precision
}
transform_eval = T.ComposedTransform([T.CenterCrop(144)])
eval_result = {key: 0 for key in metric_dict.keys()}
eval_result['loss_gm'] = 0
eval_result['loss_spinal_cord'] = 0
eval_result['auc'] = 0
eval_result['avg_surface_distance'] = 0
with torch.no_grad():
for data3D, gt_list in loader:
gt_list = torch.cat([gt.byte() for gt in gt_list], 0)
x, gt = data3D.transpose(0, 1), gt_list.transpose(0, 1)
x_max = x.max(dim=2)[0].max(dim=2)[0]
x_max = (x_max + (x_max == 0).float()).view(-1, 1, 1, 1)
x = x / x_max
true_masks = gt.cuda()
spinal_cord_mask = (torch.mean(
(true_masks > 0).float(), dim=1) > 0.5).unsqueeze(
dim=1).float()
gm_gt_mask = (torch.mean(
(true_masks == 1).float(), dim=1) > 0.5).unsqueeze(
dim=1).float()
trans_x, trans_cord_mask, trans_gm_mask = [], [], []
for i in range(x.shape[0]):
a, b, c = transform_eval(x[i], spinal_cord_mask[i],
gm_gt_mask[i])
trans_x.append(a), trans_cord_mask.append(
b), trans_gm_mask.append(c)
x, spinal_cord_mask, gm_gt_mask = torch.stack(
trans_x, dim=0).cuda(), torch.stack(
trans_cord_mask, dim=0).cuda(), torch.stack(trans_gm_mask,
dim=0).cuda()
spinal_cord_pred, _ = net_spinal_cord(x)
loss_spinal_cord = criterion(spinal_cord_pred, spinal_cord_mask)
spinal_mask_pred = (torch.sigmoid(spinal_cord_pred) >
0.5).detach().float()
local_max = (spinal_mask_pred * x).max(dim=2)[0].max(dim=2)[0]
local_min = ((1 - spinal_mask_pred) * 9999 +
spinal_mask_pred * x).min(dim=2)[0].min(dim=2)[0]
local_max = local_max.view(-1, 1, 1, 1)
local_min = local_min.view(-1, 1, 1, 1)
local_min *= (local_min < 9000).float()
x = torch.clamp(
(x - local_min) / ((local_max - local_min) +
((local_max - local_min) == 0).float()), 0,
1)
gm_pred, _ = net_gm(x) # * spinal_mask_pred
gm_pos_weight = torch.sum(spinal_cord_mask) / torch.sum(
spinal_cord_mask * gm_gt_mask)
if torch.isinf(gm_pos_weight) or torch.isnan(gm_pos_weight):
gm_pos_weight = torch.tensor(1.).cuda()
loss_gm = F.binary_cross_entropy_with_logits(
gm_pred * spinal_mask_pred,
gm_gt_mask,
pos_weight=gm_pos_weight)
gm_pred_mask = torch.sigmoid(gm_pred * spinal_mask_pred) > 0.5
eval_result['loss_gm'] += loss_gm.item()
eval_result['loss_spinal_cord'] += loss_spinal_cord.item()
eval_result['auc'] += skmetrics.roc_auc_score(
gm_gt_mask.view(-1).cpu().numpy(),
gm_pred.view(-1).cpu().numpy())
surface_dis = surface_distance.compute_surface_distances(
gm_gt_mask.cpu().squeeze().numpy().astype(np.bool),
gm_pred_mask.cpu().squeeze().numpy().astype(np.bool),
spacing_mm=resolution)
eval_result['avg_surface_distance'] += np.mean(
surface_distance.compute_average_surface_distance(surface_dis))
# gm_pred_full_size = (torch.sigmoid(gm_pred) * spinal_mask_pred) > 0.5
# writer.add_images('raw_data', x, global_step=epoch)
# writer.add_images('SpinalList_Pred', spinal_mask_pred, global_step=epoch)
# writer.add_images('SpinalList_GT', spinal_cord_mask, global_step=epoch)
# writer.add_images('GmMask_Pred', gm_pred_mask, global_step=epoch)
# writer.add_images('GmMask_GT', gm_gt_mask, global_step=epoch)
for key, metric_func in metric_dict.items():
eval_result[key] += metric_func(gm_pred_mask, gm_gt_mask)
for key, val in eval_result.items():
eval_result[key] = val / len(loader)
# writer.add_scalar('eval/' + key, eval_result[key], global_step=epoch)
info_list = [{
'name': key,
'val': value
} for key, value in eval_result.items()]
logger.log_epoch_info(info_list, epoch=epoch)
return eval_result
def test_two_squares_shifted_by_one_pixel(self):
# We make sure we do not have active pixels on the border of the image,
# because this will add additional 2D surfaces on the border of the image
# because the image is padded with background.
mask_gt = np.asarray([
[0, 0, 0, 0, 0, 0],
[0, 1, 1, 0, 0, 0],
[0, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
],
dtype=np.bool)
mask_pred = np.asarray([
[0, 0, 0, 0, 0, 0],
[0, 1, 1, 0, 0, 0],
[0, 1, 1, 0, 0, 0],
[0, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
],
dtype=np.bool)
vertical = 2
horizontal = 1
diag = 0.5 * math.sqrt(horizontal**2 + vertical**2)
surface_distances = surface_distance.compute_surface_distances(
mask_gt, mask_pred, spacing_mm=(vertical, horizontal))
# We go from top left corner, clockwise to describe the surfaces and
# distances. The 2 surfaces are:
#
# /-\ /-\
# | | | |
# \-/ | |
# \-/
expected_surfel_areas_gt = np.asarray([
diag, horizontal, diag, vertical, diag, horizontal, diag, vertical
])
expected_surfel_areas_pred = np.asarray([
diag, horizontal, diag, vertical, vertical, diag, horizontal, diag,
vertical, vertical
])
expected_distances_gt_to_pred = np.asarray([0] * 5 + [horizontal] +
[0] * 2)
expected_distances_pred_to_gt = np.asarray([0] * 5 + [vertical] * 3 +
[0] * 2)
# We sort these using the same sorting algorithm
(expected_distances_gt_to_pred,
expected_surfel_areas_gt) = (metrics._sort_distances_surfels(
expected_distances_gt_to_pred, expected_surfel_areas_gt))
(expected_distances_pred_to_gt,
expected_surfel_areas_pred) = (metrics._sort_distances_surfels(
expected_distances_pred_to_gt, expected_surfel_areas_pred))
expected_distances = {
'surfel_areas_gt': expected_surfel_areas_gt,
'surfel_areas_pred': expected_surfel_areas_pred,
'distances_gt_to_pred': expected_distances_gt_to_pred,
'distances_pred_to_gt': expected_distances_pred_to_gt,
}
self.assertEqual(len(expected_distances), len(surface_distances))
for key, expected_value in expected_distances.items():
np.testing.assert_array_equal(expected_value,
surface_distances[key])
self._assert_metrics(
surface_distances,
mask_gt,
mask_pred,
expected_average_surface_distance=(
surface_distance.compute_average_surface_distance(
expected_distances)),
expected_hausdorff_100=(surface_distance.compute_robust_hausdorff(
expected_distances, 100)),
expected_hausdorff_95=surface_distance.compute_robust_hausdorff(
expected_distances, 95),
expected_surface_overlap_at_1mm=(
surface_distance.compute_surface_overlap_at_tolerance(
expected_distances, tolerance_mm=1)),
expected_surface_dice_at_1mm=(
surface_distance.compute_surface_dice_at_tolerance(
surface_distances, tolerance_mm=1)),
expected_volumetric_dice=(
surface_distance.compute_dice_coefficient(mask_gt, mask_pred)))
y_true = te_mask.reshape(
te_mask.shape[0] * te_mask.shape[1] * te_mask.shape[2], 1)
y_pred = np.where(y_scores > 0.8, 1, 0)
y_true = np.where(y_true > 0.5, 1, 0)
# Surface Distance Metrics:
hausdorff_list = np.array([0 for i in range(predictions.shape[0])])
avg_surf_dist = np.array(
[np.array([0, 0]) for i in range(predictions.shape[0])])
for i in range(len(predictions)):
surface_dist_dict = sd.compute_surface_distances(
predictions[i].astype(bool), Estimated_lung[i].astype(bool), [1, 1])
hausdorff_list[i] = sd.compute_robust_hausdorff(surface_dist_dict, 1)
tmp_avg = sd.compute_average_surface_distance(surface_dist_dict)
avg_surf_dist[i, 0] = tmp_avg[0]
avg_surf_dist[i, 1] = tmp_avg[1]
print("\nMean of Hausdorff distances: " + str(np.mean(hausdorff_list)))
print("\nMean of Average Surface Distances (Original to predicted): " +
str(np.mean(avg_surf_dist[:, 0])))
print("\nMean of Average Surface Distances (Predicted to original): " +
str(np.mean(avg_surf_dist[:, 1])))
# Area under the ROC curve
fpr, tpr, thresholds = roc_curve((y_true), y_pred)
AUC_ROC = roc_auc_score(y_true, y_pred)
print("\nArea under the ROC curve: " + str(AUC_ROC))
roc_curve = plt.figure()
plt.plot(fpr, tpr, '-', label='Area Under the Curve (AUC = %0.4f)' % AUC_ROC)