def test_hausdorff_empty(): empty = np.zeros((0, 2), dtype=bool) non_empty = np.zeros((3, 2), dtype=bool) assert hausdorff_distance(empty, non_empty) == 0. assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()]) assert hausdorff_distance(non_empty, empty) == 0. assert_array_equal(hausdorff_pair(non_empty, empty), [(), ()]) assert hausdorff_distance(empty, non_empty) == 0. assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()])
def test_gallery(): shape = (60, 60) # Create a diamond-like shape where the four corners form the 1st set # of points x_diamond = 30 y_diamond = 30 r = 10 plt_x = [0, 1, 0, -1] plt_y = [1, 0, -1, 0] set_ax = [(x_diamond + r * x) for x in plt_x] set_ay = [(y_diamond + r * y) for y in plt_y] # Create a kite-like shape where the four corners form the 2nd set of # points x_kite = 30 y_kite = 30 x_r = 15 y_r = 20 set_bx = [(x_kite + x_r * x) for x in plt_x] set_by = [(y_kite + y_r * y) for y in plt_y] # Set up the data to compute the Hausdorff distance coords_a = np.zeros(shape, dtype=bool) coords_b = np.zeros(shape, dtype=bool) for x, y in zip(set_ax, set_ay): coords_a[(x, y)] = True for x, y in zip(set_bx, set_by): coords_b[(x, y)] = True # Test the Hausdorff function on the coordinates # Should return 10, the distance between the furthest tip of the kite and # its closest point on the diamond, which is the furthest someone can make # you travel to encounter your nearest neighboring point on the other set. assert_almost_equal(hausdorff_distance(coords_a, coords_b), 10.0) # There are two pairs of points ((30, 20), (30, 10) or (30, 40), (30, 50)), # that are Hausdorff distance apart. This tests for either of them. hd_points = hausdorff_pair(coords_a, coords_b) assert (np.equal(hd_points, ((30, 20), (30, 10))).all() or np.equal(hd_points, ((30, 40), (30, 50))).all()) # Test the Modified Hausdorff function on the coordinates # Should return 7.5. assert_almost_equal( hausdorff_distance(coords_a, coords_b, method="modified"), 7.5)
def test_hausdorff_region_different_points(points_a, points_b): shape = (7, 7) coords_a = np.zeros(shape, dtype=bool) coords_b = np.zeros(shape, dtype=bool) coords_a[points_a] = True coords_b[points_b] = True dist = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b))) d = distance.cdist([points_a], [points_b]) dist_modified = max(np.mean(np.min(d, axis=0)), np.mean(np.min(d, axis=1))) assert_almost_equal(hausdorff_distance(coords_a, coords_b), dist) assert_array_equal(hausdorff_pair(coords_a, coords_b), (points_a, points_b)) assert_almost_equal( hausdorff_distance(coords_a, coords_b, method="modified"), dist_modified)
def test_hausdorff_region_different_points(points_a, points_b): shape = (7, 7) coords_a = np.zeros(shape, dtype=np.bool) coords_b = np.zeros(shape, dtype=np.bool) coords_a[points_a] = True coords_b[points_b] = True distance = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b))) assert_almost_equal(hausdorff_distance(coords_a, coords_b), distance)
def test_hausdorff_simple(): points_a = (3, 0) points_b = (6, 0) shape = (7, 1) coords_a = np.zeros(shape, dtype=np.bool) coords_b = np.zeros(shape, dtype=np.bool) coords_a[points_a] = True coords_b[points_b] = True distance = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b))) assert_almost_equal(hausdorff_distance(coords_a, coords_b), distance)
def test_hausdorff_region_single(points_a, points_b): shape = (5, 5) coords_a = np.zeros(shape, dtype=bool) coords_b = np.zeros(shape, dtype=bool) coords_a[points_a] = True coords_b[points_b] = True dist = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b))) assert_almost_equal(hausdorff_distance(coords_a, coords_b), dist) assert_array_equal(hausdorff_pair(coords_a, coords_b), (points_a, points_b))
def test_3d_hausdorff_region(points_a, points_b): hausdorff_distances_list = [] shape = (3, 3, 3) coords_a = np.zeros(shape, dtype=np.bool) coords_b = np.zeros(shape, dtype=np.bool) coords_a[points_a] = True coords_b[points_b] = True distance = np.sqrt(sum((ca - cb)**2 for ca, cb in zip(points_a, points_b))) hausdorff_distance_3d = hausdorff_distance(coords_a, coords_b) assert_almost_equal(hausdorff_distance_3d, distance) hausdorff_distances_list.append(hausdorff_distance_3d)
def test_hausdorff_metrics_match(): # Test that Hausdorff distance is the Euclidean distance between Hausdorff # pair points_a = (3, 0) points_b = (6, 0) shape = (7, 1) coords_a = np.zeros(shape, dtype=bool) coords_b = np.zeros(shape, dtype=bool) coords_a[points_a] = True coords_b[points_b] = True assert_array_equal(hausdorff_pair(coords_a, coords_b), (points_a, points_b)) euclidean_distance = distance.euclidean(points_a, points_b) assert_almost_equal(euclidean_distance, hausdorff_distance(coords_a, coords_b))
def get_hausdorff_distance(self, array1, array2): hausdorff = [[] for i in range(array1.shape[1])] # per target class for batch_idx in range(array1.shape[0]): for channel_idx in range(array1.shape[1]): hd = hausdorff_distance( np.array(array1[batch_idx, channel_idx].cpu()), np.array(array2[batch_idx, channel_idx].cpu())) if not np.isinf(hd): hausdorff[channel_idx].append(hd) for idx, i in enumerate(hausdorff): hausdorff[idx] = sum(i) / len( i ) # [ [class_1], [class_2 ]] --> [ class_1_mean, class_2_mean] if len(hausdorff) > 1: mean = np.mean(hausdorff) hausdorff.append(mean) # add mean hd return hausdorff
def _set_threshold(pred, target, optimize_for: str = "dice"): # set threshold which minimizes hausdorff distance, handles outliers possible_values = np.arange(0.20, 1, 0.05) threshold = False best = 10000000000000 for t in possible_values: tmp_pred = deepcopy(pred) tmp_pred[tmp_pred >= t] = 1 # doesnt matter for hausdorff as all non zeros count tmp_pred[tmp_pred < t] = 0 if optimize_for == "dice": metric = DiceLoss.dice_loss(tmp_pred, target, return_loss=True) # to conform with less is better of hausdorff elif optimize_for == "hausdorff": metric = hausdorff_distance(tmp_pred, target) if metric < best: best = metric threshold = t del tmp_pred return threshold
def test_gallery(): shape = (60, 60) # Create a diamond-like shape where the four corners form the 1st set # of points x_diamond = 30 y_diamond = 30 r = 10 plt_x = [0, 1, 0, -1] plt_y = [1, 0, -1, 0] set_ax = [(x_diamond + r * x) for x in plt_x] set_ay = [(y_diamond + r * y) for y in plt_y] # Create a kite-like shape where the four corners form the 2nd set of # points x_kite = 30 y_kite = 30 x_r = 15 y_r = 20 set_bx = [(x_kite + x_r * x) for x in plt_x] set_by = [(y_kite + y_r * y) for y in plt_y] # Set up the data to compute the hausdorff distance coords_a = np.zeros(shape, dtype=np.bool) coords_b = np.zeros(shape, dtype=np.bool) for x, y in zip(set_ax, set_ay): coords_a[(x, y)] = True for x, y in zip(set_bx, set_by): coords_b[(x, y)] = True # Test the hausdorff function on the coordinates # Should return 10, the distance between the furthest tip of the kite and # its closest point on the diamond, which is the furthest someone can make # you travel to encounter your nearest neighboring point on the other set. assert_almost_equal(hausdorff_distance(coords_a, coords_b), 10.)
def test_hausdorff_empty(): empty = np.zeros((0, 2), dtype=bool) non_empty = np.zeros((3, 2), dtype=bool) assert hausdorff_distance(empty, non_empty) == 0.0 # standard Hausdorff assert (hausdorff_distance(empty, non_empty, method="modified") == 0.0) # modified Hausdorff with expected_warnings(["One or both of the images is empty"]): assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()]) assert hausdorff_distance(non_empty, empty) == 0.0 # standard Hausdorff assert (hausdorff_distance(non_empty, empty, method="modified") == 0.0) # modified Hausdorff with expected_warnings(["One or both of the images is empty"]): assert_array_equal(hausdorff_pair(non_empty, empty), [(), ()]) assert hausdorff_distance(empty, non_empty) == 0.0 # standard Hausdorff assert (hausdorff_distance(empty, non_empty, method="modified") == 0.0) # modified Hausdorff with expected_warnings(["One or both of the images is empty"]): assert_array_equal(hausdorff_pair(empty, non_empty), [(), ()])
def _set_threshold(pred, target, optimize_for: str = "dice"): # set threshold which minimizes hausdorff distance, handles outliers if pred.shape[1] == 2: # 2 sturctures use argmax return [None, False ] # doesnt matter isnt used but needs same len of channels possible_values = np.arange(0.20, 1, 0.05) threshold = [False for _ in range(pred.shape[1])] for channel_idx in range(pred.shape[1]): best = 10000000000000 for t in possible_values: tmp_pred = deepcopy( torch.unsqueeze(pred[:, channel_idx], dim=0)) # (1,1,x,y,z) tmp_target = deepcopy( torch.unsqueeze(target[:, channel_idx], dim=0)) # (1,1,x,y,z) tmp_pred[ tmp_pred >= t] = 1 # doesnt matter for hausdorff as all non zeros count tmp_pred[tmp_pred < t] = 0 if optimize_for == "dice": metric = DiceLoss.dice_loss( tmp_pred, tmp_target, return_loss=True ) # to conform with "less is better" of hausdorff elif optimize_for == "hausdorff": metric = hausdorff_distance( np.array(tmp_pred[0, 0].cpu()), np.array(tmp_target[0, 0].cpu())) # (x,y,z) if metric < best: best = metric threshold[channel_idx] = t return threshold
def test_hausdorff_empty(): empty = np.zeros((0, 2), dtype=np.bool) non_empty = np.zeros((3, 2), dtype=np.bool) assert hausdorff_distance(empty, non_empty) == 0. assert hausdorff_distance(non_empty, empty) == 0. assert hausdorff_distance(empty, empty) == 0.
def time_hausdorff(self): metrics.hausdorff_distance(self.coords_a, self.coords_b)
set_bx = [(x_kite + x_r * x) for x in plt_x] set_by = [(y_kite + y_r * y) for y in plt_y] plt.plot(set_bx, set_by, 'og') # Set up the data to compute the hausdorff distance coords_a = np.zeros(shape, dtype=bool) coords_b = np.zeros(shape, dtype=bool) for x, y in zip(set_ax, set_ay): coords_a[(x, y)] = True for x, y in zip(set_bx, set_by): coords_b[(x, y)] = True # Call the hausdorff function on the coordinates metrics.hausdorff_distance(coords_a, coords_b) # Plot the lines that shows the length of the hausdorff distance x_line = [30, 30] y_line = [20, 10] plt.plot(x_line, y_line, 'y') x_line = [30, 30] y_line = [40, 50] plt.plot(x_line, y_line, 'y') # Plot circles to show that at this distance, the hausdorff distance can # travel to its nearest neighbor (in this case, from the kite to diamond) ax.add_artist(plt.Circle((30, 10), 10, color='y', fill=None)) ax.add_artist(plt.Circle((30, 50), 10, color='y', fill=None)) ax.add_artist(plt.Circle((15, 30), 10, color='y', fill=None))
def time_modified_hausdorff_distance(self): metrics.hausdorff_distance(self.coords_a, self.coords_b, method="modified")
def __call__(self, y_true_mask, y_pred_mask): check_mask(y_true_mask) check_mask(y_pred_mask) score = metrics.hausdorff_distance(y_true_mask, y_pred_mask) return score
def hausdorff(img1, img2): return metrics.hausdorff_distance(img1, img2)
def compute_metrics_for_all_cubes(self, inference_full_image=True): cubes_to_use = [] dump_tensors() torch.cuda.ipc_collect() torch.cuda.empty_cache() torch.cuda.ipc_collect() torch.cuda.empty_cache() dump_tensors() torch.cuda.empty_cache() if "lidc" in self.dataset_name: return if hasattr(self.trainer, "model"): del self.trainer.model del self.trainer sleep(20) self.trainer = Trainer(config=self.config, dataset=None) dump_tensors() torch.cuda.ipc_collect() torch.cuda.empty_cache() dump_tensors() dump_tensors() torch.cuda.ipc_collect() torch.cuda.empty_cache() dump_tensors() self.trainer.load_model(from_path=True, path=self.model_path, phase="sup", ensure_sup_is_completed=True) if inference_full_image is False: print("PATCHING Will be Done") full_cubes_used_for_testing = self.get_all_cubes_which_were_used_for_testing() full_cubes_used_for_training = self.get_all_cubes_which_were_used_for_training() cubes_to_use.extend(full_cubes_used_for_testing) cubes_to_use.extend(full_cubes_used_for_training) cubes_to_use_path = [os.path.join(self.dataset_dir, i) for i in cubes_to_use] label_cubes_of_cubes_to_use_path = [os.path.join(self.dataset_labels_dir, i) for i in cubes_to_use] metric_dict = dict() ( dice_logits_test, dice_logits_train, dice_binary_test, dice_binary_train, jaccard_test, jaccard_train, hausdorff_test, hausdorff_train, ) = ([], [], [], [], [], [], [], []) for idx, cube_path in enumerate(cubes_to_use_path): np_array = self._load_cube_to_np_array(cube_path) # (x,y,z) self.original_cube_dimensions = np_array.shape if sum([i for i in np_array.shape]) > 550 and self.two_dim is False: inference_full_image = False if self.dataset_name.lower() in ("task04_sup", "task01_sup", "cellari_heart_sup_10_192", "cellari_heart_sup"): if self.tried is False: inference_full_image = True else: inference_full_image = False if inference_full_image is False: print("CUBE TOO BIG, PATCHING") patcher = Patcher(np_array, two_dim=self.two_dim) with torch.no_grad(): self.trainer.model.eval() for patch_idx, patch in patcher: patch = torch.unsqueeze(patch, 0) # (1,C,H,W or 1) -> (1,1,C,H,W or 1) if self.config.model.lower() in ( "vnet_mg", "unet_3d", "unet_acs", "unet_acs_axis_aware_decoder", "unet_acs_with_cls", ): patch, pad_tuple = pad_if_necessary_one_array(patch, return_pad_tuple=True) pred = self.trainer.model(patch) assert pred.shape == patch.shape, "{} vs {}".format(pred.shape, patch.shape) # need to then unpad to reconstruct if self.two_dim is True: raise RuntimeError("SHOULD NOT BE USED HERE") pred = self._unpad_3d_array(pred, pad_tuple) pred = torch.squeeze(pred, dim=0) # (1, 1, C,H,W) -> (1,C,H,W) # pred_mask = self._make_pred_mask_from_pred(pred) patcher.predicitons_to_reconstruct_from[ :, patch_idx ] = pred # update array in patcher that will construct full cube predicted mask del pred dump_tensors() torch.cuda.ipc_collect() torch.cuda.empty_cache() dump_tensors() pred_mask_full_cube = patcher.get_pred_mask_full_cube() else: full_cube_tensor = torch.Tensor(np_array) full_cube_tensor = torch.unsqueeze(full_cube_tensor, 0) # (C,H,W) -> (1,C,H,W) full_cube_tensor = torch.unsqueeze(full_cube_tensor, 0) # (1,C,H,W) -> (1,1,C,H,W) with torch.no_grad(): self.trainer.model.eval() if self.two_dim is False: if self.config.model.lower() in ( "vnet_mg", "unet_3d", "unet_acs", "unet_acs_axis_aware_decoder", "unet_acs_with_cls", ): full_cube_tensor, pad_tuple = pad_if_necessary_one_array(full_cube_tensor, return_pad_tuple=True) try: p = self.trainer.model(full_cube_tensor) p.to("cpu") pred = p del p dump_tensors() torch.cuda.ipc_collect() torch.cuda.empty_cache() dump_tensors() torch.cuda.empty_cache() pred = self._unpad_3d_array(pred, pad_tuple) pred = torch.squeeze(pred, dim=0) # (1, 1, C,H,W) -> (1,C,H,W) pred = torch.squeeze(pred, dim=0) pred_mask_full_cube = pred # self._make_pred_mask_from_pred(pred) torch.cuda.ipc_collect() torch.cuda.empty_cache() del pred except RuntimeError as e: if "out of memory" in str(e) or "cuDNN error: CUDNN_STATUS_NOT_SUPPORTED" in str(e): print("TOO BIG FOR MEMORY, DEFAULTING TO PATCHING") # exit(0) dump_tensors() torch.cuda.ipc_collect() torch.cuda.empty_cache() dump_tensors() self.tried = True res = self.compute_metrics_for_all_cubes(inference_full_image=False) return res else: pred_mask_full_cube = torch.zeros(self.original_cube_dimensions) for z_idx in range(full_cube_tensor.size()[-1]): tensor_slice = full_cube_tensor[..., z_idx] # SLICE : (1,1,C,H,W) -> (1,1,C,H) assert tensor_slice.shape == (1, 1, self.original_cube_dimensions[0], self.original_cube_dimensions[1]) pred = self.trainer.model(tensor_slice) pred = torch.squeeze(pred, dim=0) # (1, 1, C,H) -> (1,C,H) pred = torch.squeeze(pred, dim=0) # (1,C,H) -> (C,H) pred_mask_slice = pred # self._make_pred_mask_from_pred(pred) pred_mask_full_cube[..., z_idx] = pred_mask_slice full_cube_label_tensor = torch.Tensor(self._load_cube_to_np_array(label_cubes_of_cubes_to_use_path[idx])) full_cube_label_tensor = self.adjust_label_cube_acording_to_dataset(full_cube_label_tensor) pred_mask_full_cube = pred_mask_full_cube.to("cpu") threshold = self._set_threshold(pred_mask_full_cube, full_cube_label_tensor) pred_mask_full_cube_binary = self._make_pred_mask_from_pred(pred_mask_full_cube, threshold=threshold) dice_score_soft = float(DiceLoss.dice_loss(pred_mask_full_cube, full_cube_label_tensor, return_loss=False)) dice_score_binary = float(DiceLoss.dice_loss(pred_mask_full_cube_binary, full_cube_label_tensor, return_loss=False)) hausdorff = hausdorff_distance(np.array(pred_mask_full_cube_binary), np.array(full_cube_label_tensor)) x_flat = pred_mask_full_cube_binary.contiguous().view(-1) y_flat = full_cube_label_tensor.contiguous().view(-1) x_flat = x_flat.cpu() y_flat = y_flat.cpu() jac_score = jaccard_score(y_flat, x_flat) if idx < len(full_cubes_used_for_testing): dice_logits_test.append(dice_score_soft) dice_binary_test.append(dice_score_binary) jaccard_test.append(jac_score) hausdorff_test.append(hausdorff) else: dice_logits_train.append(dice_score_soft) dice_binary_train.append(dice_score_binary) jaccard_train.append(jac_score) hausdorff_train.append(hausdorff) dump_tensors() torch.cuda.ipc_collect() torch.cuda.empty_cache() dump_tensors() sleep(10) print(idx) avg_jaccard_test = sum(jaccard_test) / len(jaccard_test) avg_jaccard_train = sum(jaccard_train) / len(jaccard_train) avg_dice_test_soft = sum(dice_logits_test) / len(dice_logits_test) avg_dice_test_binary = sum(dice_binary_test) / len(dice_binary_test) avg_dice_train_soft = sum(dice_logits_train) / len(dice_logits_train) avg_dice_train_binary = sum(dice_binary_train) / len(dice_binary_train) avg_hausdorff_train = sum(hausdorff_train) / len(hausdorff_train) avg_hausdorff_test = sum(hausdorff_test) / len(hausdorff_test) metric_dict["dice_test_soft"] = avg_dice_test_soft metric_dict["dice_test_binary"] = avg_dice_test_binary metric_dict["dice_train_soft"] = avg_dice_train_soft metric_dict["dice_train_binary"] = avg_dice_train_binary metric_dict["jaccard_test"] = avg_jaccard_test metric_dict["jaccard_train"] = avg_jaccard_train metric_dict["hausdorff_test"] = avg_hausdorff_test metric_dict["hausdorff_train"] = avg_hausdorff_train return metric_dict