def test_hausdorff_empty():
empty = np.zeros((0, 2), dtype=bool)
non_empty = np.zeros((3, 2), dtype=bool)
assert hausdorff_distance(empty, non_empty) == 0.
assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()])
assert hausdorff_distance(non_empty, empty) == 0.
assert_array_equal(hausdorff_pair(non_empty, empty), [(), ()])
assert hausdorff_distance(empty, non_empty) == 0.
assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()])
def test_gallery():
shape = (60, 60)
# Create a diamond-like shape where the four corners form the 1st set
# of points
x_diamond = 30
y_diamond = 30
r = 10
plt_x = [0, 1, 0, -1]
plt_y = [1, 0, -1, 0]
set_ax = [(x_diamond + r * x) for x in plt_x]
set_ay = [(y_diamond + r * y) for y in plt_y]
# Create a kite-like shape where the four corners form the 2nd set of
# points
x_kite = 30
y_kite = 30
x_r = 15
y_r = 20
set_bx = [(x_kite + x_r * x) for x in plt_x]
set_by = [(y_kite + y_r * y) for y in plt_y]
# Set up the data to compute the Hausdorff distance
coords_a = np.zeros(shape, dtype=bool)
coords_b = np.zeros(shape, dtype=bool)
for x, y in zip(set_ax, set_ay):
coords_a[(x, y)] = True
for x, y in zip(set_bx, set_by):
coords_b[(x, y)] = True
# Test the Hausdorff function on the coordinates
# Should return 10, the distance between the furthest tip of the kite and
# its closest point on the diamond, which is the furthest someone can make
# you travel to encounter your nearest neighboring point on the other set.
assert_almost_equal(hausdorff_distance(coords_a, coords_b), 10.0)
# There are two pairs of points ((30, 20), (30, 10) or (30, 40), (30, 50)),
# that are Hausdorff distance apart. This tests for either of them.
hd_points = hausdorff_pair(coords_a, coords_b)
assert (np.equal(hd_points, ((30, 20), (30, 10))).all()
or np.equal(hd_points, ((30, 40), (30, 50))).all())
# Test the Modified Hausdorff function on the coordinates
# Should return 7.5.
assert_almost_equal(
hausdorff_distance(coords_a, coords_b, method="modified"), 7.5)
def test_hausdorff_region_different_points(points_a, points_b):
shape = (7, 7)
coords_a = np.zeros(shape, dtype=bool)
coords_b = np.zeros(shape, dtype=bool)
coords_a[points_a] = True
coords_b[points_b] = True
dist = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b)))
d = distance.cdist([points_a], [points_b])
dist_modified = max(np.mean(np.min(d, axis=0)), np.mean(np.min(d, axis=1)))
assert_almost_equal(hausdorff_distance(coords_a, coords_b), dist)
assert_array_equal(hausdorff_pair(coords_a, coords_b),
(points_a, points_b))
assert_almost_equal(
hausdorff_distance(coords_a, coords_b, method="modified"),
dist_modified)
def test_hausdorff_region_different_points(points_a, points_b):
shape = (7, 7)
coords_a = np.zeros(shape, dtype=np.bool)
coords_b = np.zeros(shape, dtype=np.bool)
coords_a[points_a] = True
coords_b[points_b] = True
distance = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b)))
assert_almost_equal(hausdorff_distance(coords_a, coords_b), distance)
def test_hausdorff_simple():
points_a = (3, 0)
points_b = (6, 0)
shape = (7, 1)
coords_a = np.zeros(shape, dtype=np.bool)
coords_b = np.zeros(shape, dtype=np.bool)
coords_a[points_a] = True
coords_b[points_b] = True
distance = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b)))
assert_almost_equal(hausdorff_distance(coords_a, coords_b), distance)
def test_hausdorff_region_single(points_a, points_b):
shape = (5, 5)
coords_a = np.zeros(shape, dtype=bool)
coords_b = np.zeros(shape, dtype=bool)
coords_a[points_a] = True
coords_b[points_b] = True
dist = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b)))
assert_almost_equal(hausdorff_distance(coords_a, coords_b), dist)
assert_array_equal(hausdorff_pair(coords_a, coords_b),
(points_a, points_b))
def test_3d_hausdorff_region(points_a, points_b):
hausdorff_distances_list = []
shape = (3, 3, 3)
coords_a = np.zeros(shape, dtype=np.bool)
coords_b = np.zeros(shape, dtype=np.bool)
coords_a[points_a] = True
coords_b[points_b] = True
distance = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b)))
hausdorff_distance_3d = hausdorff_distance(coords_a, coords_b)
assert_almost_equal(hausdorff_distance_3d, distance)
hausdorff_distances_list.append(hausdorff_distance_3d)
def test_hausdorff_metrics_match():
# Test that Hausdorff distance is the Euclidean distance between Hausdorff
# pair
points_a = (3, 0)
points_b = (6, 0)
shape = (7, 1)
coords_a = np.zeros(shape, dtype=bool)
coords_b = np.zeros(shape, dtype=bool)
coords_a[points_a] = True
coords_b[points_b] = True
assert_array_equal(hausdorff_pair(coords_a, coords_b),
(points_a, points_b))
euclidean_distance = distance.euclidean(points_a, points_b)
assert_almost_equal(euclidean_distance,
hausdorff_distance(coords_a, coords_b))
def get_hausdorff_distance(self, array1, array2):
hausdorff = [[] for i in range(array1.shape[1])] # per target class
for batch_idx in range(array1.shape[0]):
for channel_idx in range(array1.shape[1]):
hd = hausdorff_distance(
np.array(array1[batch_idx, channel_idx].cpu()),
np.array(array2[batch_idx, channel_idx].cpu()))
if not np.isinf(hd):
hausdorff[channel_idx].append(hd)
for idx, i in enumerate(hausdorff):
hausdorff[idx] = sum(i) / len(
i
) # [ [class_1], [class_2 ]] --> [ class_1_mean, class_2_mean]
if len(hausdorff) > 1:
mean = np.mean(hausdorff)
hausdorff.append(mean) # add mean hd
return hausdorff
def _set_threshold(pred, target, optimize_for: str = "dice"):
# set threshold which minimizes hausdorff distance, handles outliers
possible_values = np.arange(0.20, 1, 0.05)
threshold = False
best = 10000000000000
for t in possible_values:
tmp_pred = deepcopy(pred)
tmp_pred[tmp_pred >= t] = 1 # doesnt matter for hausdorff as all non zeros count
tmp_pred[tmp_pred < t] = 0
if optimize_for == "dice":
metric = DiceLoss.dice_loss(tmp_pred, target, return_loss=True) # to conform with less is better of hausdorff
elif optimize_for == "hausdorff":
metric = hausdorff_distance(tmp_pred, target)
if metric < best:
best = metric
threshold = t
del tmp_pred
return threshold
def test_gallery():
shape = (60, 60)
# Create a diamond-like shape where the four corners form the 1st set
# of points
x_diamond = 30
y_diamond = 30
r = 10
plt_x = [0, 1, 0, -1]
plt_y = [1, 0, -1, 0]
set_ax = [(x_diamond + r * x) for x in plt_x]
set_ay = [(y_diamond + r * y) for y in plt_y]
# Create a kite-like shape where the four corners form the 2nd set of
# points
x_kite = 30
y_kite = 30
x_r = 15
y_r = 20
set_bx = [(x_kite + x_r * x) for x in plt_x]
set_by = [(y_kite + y_r * y) for y in plt_y]
# Set up the data to compute the hausdorff distance
coords_a = np.zeros(shape, dtype=np.bool)
coords_b = np.zeros(shape, dtype=np.bool)
for x, y in zip(set_ax, set_ay):
coords_a[(x, y)] = True
for x, y in zip(set_bx, set_by):
coords_b[(x, y)] = True
# Test the hausdorff function on the coordinates
# Should return 10, the distance between the furthest tip of the kite and
# its closest point on the diamond, which is the furthest someone can make
# you travel to encounter your nearest neighboring point on the other set.
assert_almost_equal(hausdorff_distance(coords_a, coords_b), 10.)
def test_hausdorff_empty():
empty = np.zeros((0, 2), dtype=bool)
non_empty = np.zeros((3, 2), dtype=bool)
assert hausdorff_distance(empty, non_empty) == 0.0 # standard Hausdorff
assert (hausdorff_distance(empty, non_empty,
method="modified") == 0.0) # modified Hausdorff
with expected_warnings(["One or both of the images is empty"]):
assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()])
assert hausdorff_distance(non_empty, empty) == 0.0 # standard Hausdorff
assert (hausdorff_distance(non_empty, empty,
method="modified") == 0.0) # modified Hausdorff
with expected_warnings(["One or both of the images is empty"]):
assert_array_equal(hausdorff_pair(non_empty, empty), [(), ()])
assert hausdorff_distance(empty, non_empty) == 0.0 # standard Hausdorff
assert (hausdorff_distance(empty, non_empty,
method="modified") == 0.0) # modified Hausdorff
with expected_warnings(["One or both of the images is empty"]):
assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()])
def _set_threshold(pred, target, optimize_for: str = "dice"):
# set threshold which minimizes hausdorff distance, handles outliers
if pred.shape[1] == 2:
# 2 sturctures use argmax
return [None, False
] # doesnt matter isnt used but needs same len of channels
possible_values = np.arange(0.20, 1, 0.05)
threshold = [False for _ in range(pred.shape[1])]
for channel_idx in range(pred.shape[1]):
best = 10000000000000
for t in possible_values:
tmp_pred = deepcopy(
torch.unsqueeze(pred[:, channel_idx],
dim=0)) # (1,1,x,y,z)
tmp_target = deepcopy(
torch.unsqueeze(target[:, channel_idx],
dim=0)) # (1,1,x,y,z)
tmp_pred[
tmp_pred >=
t] = 1 # doesnt matter for hausdorff as all non zeros count
tmp_pred[tmp_pred < t] = 0
if optimize_for == "dice":
metric = DiceLoss.dice_loss(
tmp_pred, tmp_target, return_loss=True
) # to conform with "less is better" of hausdorff
elif optimize_for == "hausdorff":
metric = hausdorff_distance(
np.array(tmp_pred[0, 0].cpu()),
np.array(tmp_target[0, 0].cpu())) # (x,y,z)
if metric < best:
best = metric
threshold[channel_idx] = t
return threshold
def test_hausdorff_empty():
empty = np.zeros((0, 2), dtype=np.bool)
non_empty = np.zeros((3, 2), dtype=np.bool)
assert hausdorff_distance(empty, non_empty) == 0.
assert hausdorff_distance(non_empty, empty) == 0.
assert hausdorff_distance(empty, empty) == 0.
def time_hausdorff(self):
metrics.hausdorff_distance(self.coords_a, self.coords_b)
set_bx = [(x_kite + x_r * x) for x in plt_x]
set_by = [(y_kite + y_r * y) for y in plt_y]
plt.plot(set_bx, set_by, 'og')
# Set up the data to compute the hausdorff distance
coords_a = np.zeros(shape, dtype=bool)
coords_b = np.zeros(shape, dtype=bool)
for x, y in zip(set_ax, set_ay):
coords_a[(x, y)] = True
for x, y in zip(set_bx, set_by):
coords_b[(x, y)] = True
# Call the hausdorff function on the coordinates
metrics.hausdorff_distance(coords_a, coords_b)
# Plot the lines that shows the length of the hausdorff distance
x_line = [30, 30]
y_line = [20, 10]
plt.plot(x_line, y_line, 'y')
x_line = [30, 30]
y_line = [40, 50]
plt.plot(x_line, y_line, 'y')
# Plot circles to show that at this distance, the hausdorff distance can
# travel to its nearest neighbor (in this case, from the kite to diamond)
ax.add_artist(plt.Circle((30, 10), 10, color='y', fill=None))
ax.add_artist(plt.Circle((30, 50), 10, color='y', fill=None))
ax.add_artist(plt.Circle((15, 30), 10, color='y', fill=None))
def time_modified_hausdorff_distance(self):
metrics.hausdorff_distance(self.coords_a,
self.coords_b,
method="modified")
def __call__(self, y_true_mask, y_pred_mask):
check_mask(y_true_mask)
check_mask(y_pred_mask)
score = metrics.hausdorff_distance(y_true_mask, y_pred_mask)
return score
def hausdorff(img1, img2):
return metrics.hausdorff_distance(img1, img2)
def compute_metrics_for_all_cubes(self, inference_full_image=True):
cubes_to_use = []
dump_tensors()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
dump_tensors()
torch.cuda.empty_cache()
if "lidc" in self.dataset_name:
return
if hasattr(self.trainer, "model"):
del self.trainer.model
del self.trainer
sleep(20)
self.trainer = Trainer(config=self.config, dataset=None)
dump_tensors()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
dump_tensors()
dump_tensors()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
dump_tensors()
self.trainer.load_model(from_path=True, path=self.model_path, phase="sup", ensure_sup_is_completed=True)
if inference_full_image is False:
print("PATCHING Will be Done")
full_cubes_used_for_testing = self.get_all_cubes_which_were_used_for_testing()
full_cubes_used_for_training = self.get_all_cubes_which_were_used_for_training()
cubes_to_use.extend(full_cubes_used_for_testing)
cubes_to_use.extend(full_cubes_used_for_training)
cubes_to_use_path = [os.path.join(self.dataset_dir, i) for i in cubes_to_use]
label_cubes_of_cubes_to_use_path = [os.path.join(self.dataset_labels_dir, i) for i in cubes_to_use]
metric_dict = dict()
(
dice_logits_test,
dice_logits_train,
dice_binary_test,
dice_binary_train,
jaccard_test,
jaccard_train,
hausdorff_test,
hausdorff_train,
) = ([], [], [], [], [], [], [], [])
for idx, cube_path in enumerate(cubes_to_use_path):
np_array = self._load_cube_to_np_array(cube_path) # (x,y,z)
self.original_cube_dimensions = np_array.shape
if sum([i for i in np_array.shape]) > 550 and self.two_dim is False:
inference_full_image = False
if self.dataset_name.lower() in ("task04_sup", "task01_sup", "cellari_heart_sup_10_192", "cellari_heart_sup"):
if self.tried is False:
inference_full_image = True
else:
inference_full_image = False
if inference_full_image is False:
print("CUBE TOO BIG, PATCHING")
patcher = Patcher(np_array, two_dim=self.two_dim)
with torch.no_grad():
self.trainer.model.eval()
for patch_idx, patch in patcher:
patch = torch.unsqueeze(patch, 0) # (1,C,H,W or 1) -> (1,1,C,H,W or 1)
if self.config.model.lower() in (
"vnet_mg",
"unet_3d",
"unet_acs",
"unet_acs_axis_aware_decoder",
"unet_acs_with_cls",
):
patch, pad_tuple = pad_if_necessary_one_array(patch, return_pad_tuple=True)
pred = self.trainer.model(patch)
assert pred.shape == patch.shape, "{} vs {}".format(pred.shape, patch.shape)
# need to then unpad to reconstruct
if self.two_dim is True:
raise RuntimeError("SHOULD NOT BE USED HERE")
pred = self._unpad_3d_array(pred, pad_tuple)
pred = torch.squeeze(pred, dim=0) # (1, 1, C,H,W) -> (1,C,H,W)
# pred_mask = self._make_pred_mask_from_pred(pred)
patcher.predicitons_to_reconstruct_from[
:, patch_idx
] = pred # update array in patcher that will construct full cube predicted mask
del pred
dump_tensors()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
dump_tensors()
pred_mask_full_cube = patcher.get_pred_mask_full_cube()
else:
full_cube_tensor = torch.Tensor(np_array)
full_cube_tensor = torch.unsqueeze(full_cube_tensor, 0) # (C,H,W) -> (1,C,H,W)
full_cube_tensor = torch.unsqueeze(full_cube_tensor, 0) # (1,C,H,W) -> (1,1,C,H,W)
with torch.no_grad():
self.trainer.model.eval()
if self.two_dim is False:
if self.config.model.lower() in (
"vnet_mg",
"unet_3d",
"unet_acs",
"unet_acs_axis_aware_decoder",
"unet_acs_with_cls",
):
full_cube_tensor, pad_tuple = pad_if_necessary_one_array(full_cube_tensor, return_pad_tuple=True)
try:
p = self.trainer.model(full_cube_tensor)
p.to("cpu")
pred = p
del p
dump_tensors()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
dump_tensors()
torch.cuda.empty_cache()
pred = self._unpad_3d_array(pred, pad_tuple)
pred = torch.squeeze(pred, dim=0) # (1, 1, C,H,W) -> (1,C,H,W)
pred = torch.squeeze(pred, dim=0)
pred_mask_full_cube = pred # self._make_pred_mask_from_pred(pred)
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
del pred
except RuntimeError as e:
if "out of memory" in str(e) or "cuDNN error: CUDNN_STATUS_NOT_SUPPORTED" in str(e):
print("TOO BIG FOR MEMORY, DEFAULTING TO PATCHING")
# exit(0)
dump_tensors()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
dump_tensors()
self.tried = True
res = self.compute_metrics_for_all_cubes(inference_full_image=False)
return res
else:
pred_mask_full_cube = torch.zeros(self.original_cube_dimensions)
for z_idx in range(full_cube_tensor.size()[-1]):
tensor_slice = full_cube_tensor[..., z_idx] # SLICE : (1,1,C,H,W) -> (1,1,C,H)
assert tensor_slice.shape == (1, 1, self.original_cube_dimensions[0], self.original_cube_dimensions[1])
pred = self.trainer.model(tensor_slice)
pred = torch.squeeze(pred, dim=0) # (1, 1, C,H) -> (1,C,H)
pred = torch.squeeze(pred, dim=0) # (1,C,H) -> (C,H)
pred_mask_slice = pred # self._make_pred_mask_from_pred(pred)
pred_mask_full_cube[..., z_idx] = pred_mask_slice
full_cube_label_tensor = torch.Tensor(self._load_cube_to_np_array(label_cubes_of_cubes_to_use_path[idx]))
full_cube_label_tensor = self.adjust_label_cube_acording_to_dataset(full_cube_label_tensor)
pred_mask_full_cube = pred_mask_full_cube.to("cpu")
threshold = self._set_threshold(pred_mask_full_cube, full_cube_label_tensor)
pred_mask_full_cube_binary = self._make_pred_mask_from_pred(pred_mask_full_cube, threshold=threshold)
dice_score_soft = float(DiceLoss.dice_loss(pred_mask_full_cube, full_cube_label_tensor, return_loss=False))
dice_score_binary = float(DiceLoss.dice_loss(pred_mask_full_cube_binary, full_cube_label_tensor, return_loss=False))
hausdorff = hausdorff_distance(np.array(pred_mask_full_cube_binary), np.array(full_cube_label_tensor))
x_flat = pred_mask_full_cube_binary.contiguous().view(-1)
y_flat = full_cube_label_tensor.contiguous().view(-1)
x_flat = x_flat.cpu()
y_flat = y_flat.cpu()
jac_score = jaccard_score(y_flat, x_flat)
if idx < len(full_cubes_used_for_testing):
dice_logits_test.append(dice_score_soft)
dice_binary_test.append(dice_score_binary)
jaccard_test.append(jac_score)
hausdorff_test.append(hausdorff)
else:
dice_logits_train.append(dice_score_soft)
dice_binary_train.append(dice_score_binary)
jaccard_train.append(jac_score)
hausdorff_train.append(hausdorff)
dump_tensors()
torch.cuda.ipc_collect()
torch.cuda.empty_cache()
dump_tensors()
sleep(10)
print(idx)
avg_jaccard_test = sum(jaccard_test) / len(jaccard_test)
avg_jaccard_train = sum(jaccard_train) / len(jaccard_train)
avg_dice_test_soft = sum(dice_logits_test) / len(dice_logits_test)
avg_dice_test_binary = sum(dice_binary_test) / len(dice_binary_test)
avg_dice_train_soft = sum(dice_logits_train) / len(dice_logits_train)
avg_dice_train_binary = sum(dice_binary_train) / len(dice_binary_train)
avg_hausdorff_train = sum(hausdorff_train) / len(hausdorff_train)
avg_hausdorff_test = sum(hausdorff_test) / len(hausdorff_test)
metric_dict["dice_test_soft"] = avg_dice_test_soft
metric_dict["dice_test_binary"] = avg_dice_test_binary
metric_dict["dice_train_soft"] = avg_dice_train_soft
metric_dict["dice_train_binary"] = avg_dice_train_binary
metric_dict["jaccard_test"] = avg_jaccard_test
metric_dict["jaccard_train"] = avg_jaccard_train
metric_dict["hausdorff_test"] = avg_hausdorff_test
metric_dict["hausdorff_train"] = avg_hausdorff_train
return metric_dict
