def dice3d(base_folder, folder, subfoldername, grp_regex, gt_folder, C): if base_folder == '': work_folder = Path(folder, subfoldername) else: work_folder = Path(base_folder,folder, subfoldername) filenames = map_(lambda p: str(p.name), work_folder.glob("*.png")) grouping_regex: Pattern = re.compile(grp_regex) stems: List[str] = [Path(filename).stem for filename in filenames] # avoid matching the extension matches: List[Match] = map_(grouping_regex.match, stems) patients: List[str] = [match.group(0) for match in matches] unique_patients: List[str] = list(set(patients)) batch_dice = torch.zeros((len(unique_patients), C)) for i, patient in enumerate(unique_patients): patient_slices = [f for f in stems if f.startswith(patient)] w,h = [256,256] n = len(patient_slices) t_seg = np.ndarray(shape=(w, h, n)) t_gt = np.ndarray(shape=(w, h, n)) for slice in patient_slices: slice_nb = int(re.split(grp_regex, slice)[1]) seg = imageio.imread(str(work_folder)+'/'+slice+'.png') gt = imageio.imread(str(gt_folder )+'/'+ slice+'.png') if seg.shape != (w, h): seg = resize_im(seg, 36) if gt.shape != (w, h): gt = resize_im(gt, 36) seg[seg == 255] = 1 t_seg[:, :, slice_nb] = seg t_gt[:, :, slice_nb] = gt t_seg = torch.from_numpy(t_seg) t_gt = torch.from_numpy(t_gt) batch_dice[i,...] = dice_batch(class2one_hot(t_seg,3), class2one_hot(t_gt,3))[0] # do not save the interclasses etcetc return batch_dice.mean(dim=0), batch_dice.std(dim=0)
def test_empty(self): t = torch.zeros(1, 100, 100) t[0, 40:60, 40:60] = 1 c = utils.class2one_hot(t, C=2) self.assertEqual(utils.dice_coef(c, c)[0, 0], 1)
def test_caca(self): t = torch.zeros(1, 100, 100) t[0, 40:60, 40:60] = 1 c = utils.class2one_hot(t, C=2) z = torch.zeros_like(c) z[0, 1, ...] = 1 self.assertEqual(utils.dice_coef(c, z, smooth=0)[0, 0], 0) # Annoying to deal with the almost equal thing
def test_empty(self): a = np.zeros((1, 256, 256)) o = utils.class2one_hot(torch.Tensor(a).type(torch.float32), C=2).numpy() res = utils.one_hot2dist(o[0]) self.assertEqual(res.shape, (2, 256, 256)) self.assertEqual(res[1].sum(), 0) self.assertEqual((res[0] <= 0).sum(), a.size)
def test_full_coverage(self): a = np.zeros((1, 256, 256)) a[:, 50:60, :] = 1 o = utils.class2one_hot(torch.Tensor(a).type(torch.float32), C=2).numpy() res = utils.one_hot2dist(o[0]) self.assertEqual(res.shape, (2, 256, 256)) self.assertEqual((res[1] <= 0).sum(), a.sum()) self.assertEqual((res[1] > 0).sum(), (1 - a).sum())
def test_closure(self): a = np.zeros((1, 256, 256)) a[:, 50:60, :] = 1 o = utils.class2one_hot(torch.Tensor(a).type(torch.float32), C=2).numpy() res = utils.one_hot2dist(o[0]) self.assertEqual(res.shape, (2, 256, 256)) neg = (res <= 0) * res self.assertEqual(neg.sum(), (o * res).sum())
def test_border(self): """ Make sure the border inside the object is 0 in the distance map """ for l in range(3, 5): a = np.zeros((1, 25, 25)) a[:, 3:3 + l, 3:3 + l] = 1 o = utils.class2one_hot(torch.Tensor(a).type(torch.float32), C=2).numpy() res = utils.one_hot2dist(o[0]) self.assertEqual(res.shape, (2, 25, 25)) border = (res[1] == 0) self.assertEqual(border.sum(), 4 * (l - 1))
def test_max_dist(self): """ The max dist for a box should be at the midle of the object, +-1 """ a = np.zeros((1, 256, 256)) a[:, 1:254, 1:254] = 1 o = utils.class2one_hot(torch.Tensor(a).type(torch.float32), C=2).numpy() res = utils.one_hot2dist(o[0]) self.assertEqual(res.shape, (2, 256, 256)) self.assertEqual(res[0].max(), 127) self.assertEqual(np.unravel_index(res[0].argmax(), (256, 256)), (127, 127)) self.assertEqual(res[1].min(), -126) self.assertEqual(np.unravel_index(res[1].argmin(), (256, 256)), (127, 127))
for _,_,files in os.walk(os.path.join(root, 'in_npy')): for file in files: image = np.load(os.path.join(root,'in_npy', file)) gt = np.load(os.path.join(root,'gt_npy', file)) #print('infering {} of shape {} and classes {}, max {} and min {} '.format( file, image.shape, np.unique(gt), image.max(), image.min())) image = image.reshape(-1, 1, 256, 256) image = torch.tensor(image, dtype=torch.float) image = Variable(image, requires_grad=True) pred = net(image) pred = F.softmax(pred, dim=1) predicted_output = probs2one_hot(pred.detach()) #np.save(os.path.join(path, 'predictions', '{}'.format(file)), pred.detach().numpy()) dice = dice_coef(predicted_output, class2one_hot(torch.tensor(gt), n_classes)) hauss = haussdorf(predicted_output, class2one_hot(torch.tensor(gt), n_classes)) #pred_label = len(np.unique(label(np.array(pred.argmax(axis = 1).detach().numpy())))) #gt_label = len(np.unique(label(gt))) pred_label = len(np.unique(label(predicted_output[0][1]))) gt_label = len(np.unique(label(class2one_hot(torch.tensor(gt), n_classes)[0][1]))) error = np.abs(pred_label - gt_label) pred_label2 = len(np.unique(label(predicted_output[0][2]))) gt_label2 = len(np.unique(label(class2one_hot(torch.tensor(gt), n_classes)[0][2]))) error2 = np.abs(pred_label2 - gt_label2) print(f"{file}, {np.float(dice[0][1])}, {np.float(hauss[0][1])},{np.float(error)} \n") print(f"{file}, {np.float(dice[0][2])}, {np.float(hauss[0][2])},{np.float(error2)} \n") fold_all_H1.write(f"{file}, {np.float(dice[0][1])}, {np.float(hauss[0][1])},{np.float(error)} \n")
from utils import dice_coef, dice_batch, save_images, tqdm_, haussdorf, probs2one_hot, class2one_hot, numpy_haussdorf root = '/media/eljurros/Transcend/Decathlone/Task02_Heart/nifty/FOLD_4/npy/val' #print(net) fieldnames = ['SLICE_ID', 'dice', 'haus', 'c_error'] folder_path = Path(root, 'characteristics') n_classes = 2 folder_path.mkdir(parents=True, exist_ok=True) fold_clean_H1 = open(os.path.join(folder_path, 'characteristics.csv'), "w") fold_clean_H1.write(f"file, size, cc, \n") for _, _, files in os.walk(os.path.join(root, 'gt_npy')): print('walking into', os.path.join(root, 'gt_npy')) for file in files: #print(file) image = np.load(os.path.join(root, 'in_npy', file)) gt = np.load(os.path.join(root, 'gt_npy', file)) gt_label = len(np.unique(label(gt))) gt_label = len( np.unique(label(class2one_hot(torch.tensor(gt), n_classes)[0][1]))) print( f"{file}, {np.float(gt_label)}, {np.float(gt_label)},{np.float(gt_label)} \n" ) fold_clean_H1.write( f"{file}, {np.float(gt_label)}, {np.float(gt_label)},{np.float(gt_label)} \n" ) fold_clean_H1.flush()
def save_slices( ct_paths, cbf_paths, cbv_paths, mtt_paths, tmax_paths, gt_paths, dest_dir: Path, shape: Tuple[int], n_augment: int, ct_dir: str = "ct", cbf_dir="cbf", cbv_dir="cbv", mtt_dir="mtt", tmax_dir="tmax", gt_dir: str = "gt", in_npy_dir="in_npy", gt_npy_dir='gt_npy' ) -> Tuple[int, int, Dict, Tuple[float, float, float]]: p_id: str = get_p_id(ct_paths) assert len( set( map_(get_p_id, [ ct_paths, cbf_paths, cbv_paths, mtt_paths, tmax_paths, gt_paths ]))) == 1 space_dict: Dict[str, Tuple[float, float]] = {} # Load the data dx, dy, dz = nib.load(str(ct_paths)).header.get_zooms() assert dx == dy ct = np.asarray(nib.load(str(ct_paths)).dataobj) cbf = np.asarray(nib.load(str(cbf_paths)).dataobj) cbv = np.asarray(nib.load(str(cbv_paths)).dataobj) mtt = np.asarray(nib.load(str(mtt_paths)).dataobj) tmax = np.asarray(nib.load(str(tmax_paths)).dataobj) gt = np.asarray(nib.load(str(gt_paths)).dataobj) assert len(set(map_(np.shape, [ct, cbf, cbv, mtt, tmax, gt]))) == 1 assert ct.dtype in [np.int32], ct.dtype assert cbf.dtype in [np.uint16], cbf.dtype assert cbv.dtype in [np.uint16], cbv.dtype assert mtt.dtype in [np.float64], mtt.dtype assert tmax.dtype in [np.float64], tmax.dtype assert gt.dtype in [np.uint8], gt.dtype pos: int = (gt == 1).sum() neg: int = (gt == 0).sum() x, y, z = ct.shape # Normalize and check data content norm_ct = norm_arr( ct) # We need to normalize the whole 3d img, not 2d slices norm_cbf = norm_arr(cbf) norm_cbv = norm_arr(cbv) norm_mtt = norm_arr(mtt) norm_tmax = norm_arr(tmax) assert 0 == norm_ct.min() and norm_ct.max() == 1, (norm_ct.min(), norm_ct.max()) assert 0 == norm_cbf.min() and norm_cbf.max() == 1, (norm_cbf.min(), norm_cbf.max()) assert 0 == norm_cbv.min() and norm_cbv.max() == 1, (norm_cbv.min(), norm_cbv.max()) assert 0 == norm_mtt.min() and norm_mtt.max() == 1, (norm_mtt.min(), norm_mtt.max()) assert 0 == norm_tmax.min() and norm_tmax.max() == 1, (norm_tmax.min(), norm_tmax.max()) one_hot_gt: Tensor = class2one_hot(torch.tensor(gt[None, ...], dtype=torch.int64), K=2)[0] assert one_hot_gt.shape == (2, 256, 256, z), one_hot_gt.shape distmap: np.ndarray = one_hot2dist(one_hot_gt.numpy(), resolution=(dx, dy, dz), dtype=np.float32) save_dir_ct: Path = Path(dest_dir, ct_dir) save_dir_cbf: Path = Path(dest_dir, cbf_dir) save_dir_cbv: Path = Path(dest_dir, cbv_dir) save_dir_mtt: Path = Path(dest_dir, mtt_dir) save_dir_tmax: Path = Path(dest_dir, tmax_dir) save_dir_gt: Path = Path(dest_dir, gt_dir) save_dir_in_npy: Path = Path(dest_dir, in_npy_dir) save_dir_gt_npy: Path = Path(dest_dir, gt_npy_dir) save_dir_distmap_npy: Path = Path(dest_dir, "3d_distmap") save_dirs = [ save_dir_ct, save_dir_cbf, save_dir_cbv, save_dir_mtt, save_dir_tmax, save_dir_gt ] for j in range(ct.shape[-1]): ct_s = norm_ct[:, :, j] cbf_s = norm_cbf[:, :, j] cbv_s = norm_cbv[:, :, j] mtt_s = norm_mtt[:, :, j] tmax_s = norm_tmax[:, :, j] gt_s = gt[:, :, j] dist_s = distmap[:, :, :, j] slices = [ct_s, cbf_s, cbv_s, mtt_s, tmax_s, gt_s] assert ct_s.shape == cbf_s.shape == cbv_s.shape, mtt_s.shape == tmax_s.shape == gt_s.shape assert gt_s.shape == dist_s[0, ...].shape, ((x, y, z), gt_s.shape, dist_s.shape) assert set(np.unique(gt_s)).issubset([0, 1]) # if gt_s.sum() > 0: # print(f"{dist_s[1].min()=} {dist_s[1].max()=}") # _, axes = plt.subplots(nrows=1, ncols=3) # axes[0].imshow(gt_s) # axes[0].set_title("GT") # tmp = axes[1].imshow(dist_s[1, ...], cmap='rainbow') # axes[1].set_title("Signed distance map") # plt.colorbar(tmp, ax=axes[1]) # tmp = axes[2].imshow(np.abs(dist_s[1, ...]), cmap='rainbow') # axes[2].set_title("Abs distance map") # plt.colorbar(tmp, ax=axes[2]) # plt.show() for k in range(n_augment + 1): if k == 0: to_save = slices else: to_save = map_(np.asarray, augment_arr(*slices)) assert to_save[0].shape == slices[0].shape, (to_save[0].shape, slices[0].shape) filename = f"{p_id}_{k:02d}_{j:04d}" space_dict[filename] = (dx, dy) for save_dir, data in zip(save_dirs, to_save): save_dir.mkdir(parents=True, exist_ok=True) if "gt" not in str(save_dir): img = (data * 255).astype(np.uint8) else: img = data.astype(np.uint8) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=UserWarning) imsave(str(Path(save_dir, filename).with_suffix(".png")), img) multimodal = np.stack( to_save[:-1]) # Do not include the ground truth assert 0 <= multimodal.min() and multimodal.max() <= 1 save_dir_in_npy.mkdir(parents=True, exist_ok=True) save_dir_gt_npy.mkdir(parents=True, exist_ok=True) np.save( Path(save_dir_in_npy, filename).with_suffix(".npy"), multimodal) np.save( Path(save_dir_gt_npy, filename).with_suffix(".npy"), to_save[-1]) save_dir_distmap_npy.mkdir(parents=True, exist_ok=True) np.save( Path(save_dir_distmap_npy, filename).with_suffix(".npy"), dist_s) return neg, pos, space_dict, (dx, dy, dz)
image = np.load(os.path.join(root, 'in_npy', file)) gt = np.load(os.path.join(root, 'gt_npy', file)) if len(np.unique(gt)) > 0: #print('infering {} of shape {} and classes {}, max {} and min {} '.format( file, image.shape, np.unique(gt), image.max(), image.min())) image = image.reshape(-1, 1, 256, 256) / 255.00 image = torch.tensor(image, dtype=torch.float) image = Variable(image, requires_grad=True) pred = net(image) pred = F.softmax(pred, dim=1).to('cpu') predicted_output = probs2one_hot(pred.detach()) #print(predicted_output.to('cpu')[:,:2:].shape,class2one_hot(torch.tensor(gt).to('cpu'), n_classes).shape ) #np.save(os.path.join(path, 'predictions', '{}'.format(file)), pred.to('cpu').detach().numpy()) #dice = dice_coef(predicted_output.to('cpu'), class2one_hot(torch.tensor(gt).to('cpu'), n_classes))[:,n,] dice = dice_coef( predicted_output.to('cpu'), class2one_hot(torch.tensor(gt).to('cpu'), n_classes))[:, n, ] hauss = haussdorf(predicted_output, class2one_hot(torch.tensor(gt), n_classes))[:, n, ] ''' fig, ax = plt.subplots() ax.imshow(np.argmax(predicted_output.detach().numpy(), axis=1)[0], cmap=plt.cm.gray) #r, contours= Get_contour_characteristics(np.argmax(predicted_output.detach().numpy(), axis=1)[0]) g, contours = Get_contour_characteristics(np.array(gt).round()) total_summ = 0 for n, contour in enumerate(contours): ax.plot(contour[:, 1].astype(int), contour[:, 0].astype(int),color='red', linewidth=-1) ax.axis('image')
print(file) image = np.array(Image.open(os.path.join(root, 'img', file))) gt = np.array(Image.open(os.path.join(root, 'gt', file))) if len(np.unique(gt)) > 0: #print('infering {} of shape {} and classes {}, max {} and min {} '.format( file, image.shape, np.unique(gt), image.max(), image.min())) image = image.reshape(-1, 5, 256, 256) / 255.00 image = torch.tensor(image, dtype=torch.float) image = Variable(image, requires_grad=True) pred = net(image) pred = F.softmax(pred, dim=1).to('cpu') predicted_output = probs2one_hot(pred.detach()) np.save(os.path.join(path, 'predictions', '{}'.format(file)), pred.to('cpu').detach().numpy()) dice = dice_coef( predicted_output.to('cpu'), class2one_hot(torch.tensor(gt).to('cpu'), n_classes)) hauss = haussdorf( predicted_output, class2one_hot(torch.tensor(gt).to('cpu'), n_classes)) plt.imsave( os.path.join(path, 'predictions', '{}.png'.format(file.split('.npy')[0])), np.argmax(predicted_output, 1)[0]) plt.imsave( os.path.join(path, 'gt', '{}.png'.format(file.split('.npy')[0])), gt) pred_label = len( np.unique(label(np.array( pred.argmax(axis=1).detach().numpy())))) gt_label = len(np.unique(label(gt)))
def save_slices( flair_path, t1_path, gt_path, dest_dir: Path, shape: Tuple[int], n_augment: int, discard_negatives: bool, flair_dir: str = "flair", t1_dir="t1", gt_dir: str = "gt", in_npy_dir="in_npy", gt_npy_dir='gt_npy' ) -> Tuple[int, int, Dict, Tuple[float, float, float]]: p_id: str = get_p_id(flair_path) assert len(set(map_(get_p_id, [flair_path, t1_path, gt_path]))) == 1 print(p_id) space_dict: Dict[str, Tuple[float, float]] = {} # Load the data dx, dy, dz = nib.load(str(flair_path)).header.get_zooms() # assert dx == dy, (dx, dy) flair = np.asarray(nib.load(str(flair_path)).dataobj) w, h, _ = flair.shape x, y, z = flair.shape t1 = np.asarray(nib.load(str(t1_path)).dataobj) gt = np.asarray(nib.load(str(gt_path)).dataobj) assert set(np.unique(gt)) <= set([0., 1., 2.]) pos: int = (gt == 1).sum() neg: int = ((gt == 0) | (gt == 2)).sum() assert len(set(map_(np.shape, [flair, t1, gt]))) == 1 assert flair.dtype in [np.float32], flair.dtype assert t1.dtype in [np.uint16], t1.dtype assert gt.dtype in [np.float32], gt.dtype # Normalize and check data content norm_flair = norm_arr( flair) # We need to normalize the whole 3d img, not 2d slices norm_t1 = norm_arr(t1) norm_gt = gt.astype(np.uint8) assert 0 == norm_flair.min() and norm_flair.max() == 1, (norm_flair.min(), norm_flair.max()) assert 0 == norm_t1.min() and norm_t1.max() == 1, (norm_t1.min(), norm_t1.max()) assert np.array_equal(np.unique(gt), np.unique(norm_gt)) resized_flair = resize(norm_flair, (256, 256, z), mode='constant', preserve_range=True, anti_aliasing=False).astype(np.float32) resized_t1 = resize(norm_t1, (256, 256, z), mode='constant', preserve_range=True, anti_aliasing=False).astype(np.float32) resized_gt = resize(norm_gt, (256, 256, z), mode='constant', preserve_range=True, anti_aliasing=False, order=0).astype(np.uint8) resized_gt[np.where( resized_gt == 2)] = 0 # Count those labels as background # Pre-compute the 3d distance map rx = dx * w / 256 ry = dy * h / 256 rz = dz # print(f"{flair.shape=}") # print(f"{(dx,dy,dz)=} {(rx,ry,rz)=}") one_hot_gt: Tensor = class2one_hot(torch.tensor(resized_gt[None, ...], dtype=torch.int64), K=2)[0] assert one_hot_gt.shape == (2, 256, 256, z), one_hot_gt.shape distmap: np.ndarray = one_hot2dist(one_hot_gt.numpy(), resolution=(rx, ry, rz), dtype=np.float32) save_dir_flair: Path = Path(dest_dir, flair_dir) save_dir_t1: Path = Path(dest_dir, t1_dir) save_dir_gt: Path = Path(dest_dir, gt_dir) save_dir_in_npy: Path = Path(dest_dir, in_npy_dir) save_dir_gt_npy: Path = Path(dest_dir, gt_npy_dir) save_dir_distmap_npy: Path = Path(dest_dir, "3d_distmap") save_dirs = [save_dir_flair, save_dir_t1, save_dir_gt] for j in range(flair.shape[-1]): flair_s = resized_flair[:, :, j] t1_s = resized_t1[:, :, j] gt_s = resized_gt[:, :, j] dist_s = distmap[:, :, :, j] # if gt_s.sum() > 0: # print(f"{dist_s.min()=} {dist_s.max()=}") # _, axes = plt.subplots(nrows=1, ncols=2) # axes[0].imshow(gt_s) # axes[0].set_title("GT") # tmp = axes[1].imshow(dist_s[1, ...]) # axes[1].set_title("Distance map") # plt.colorbar(tmp, ax=axes[1]) # plt.show() slices = [flair_s, t1_s, gt_s] assert flair_s.shape == t1_s.shape == gt_s.shape == dist_s[ 0, ...].shape, ((x, y, z), flair_s.shape, dist_s.shape) # gt_s[np.where(gt_s == 2)] = 0 # Now do that part earlier assert set(np.unique(gt_s)).issubset([0, 1]), np.unique(gt_s) if discard_negatives and (gt_s.sum() == 0): continue for k in range(n_augment + 1): if k == 0: to_save = slices else: to_save = map_(np.asarray, augment_arr(*slices)) assert to_save[0].shape == slices[0].shape, (to_save[0].shape, slices[0].shape) filename = f"{p_id}_{k:02d}_{j:04d}" space_dict[filename] = (rx, ry) for save_dir, data in zip(save_dirs, to_save): save_dir.mkdir(parents=True, exist_ok=True) if "gt" not in str(save_dir): img = (data * 255).astype(np.uint8) else: img = data.astype(np.uint8) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=UserWarning) imsave(str(Path(save_dir, filename).with_suffix(".png")), img) multimodal = np.stack( to_save[:-1]) # Do not include the ground truth assert 0 <= multimodal.min(), multimodal.min() assert multimodal.max() <= 1, multimodal.max() save_dir_in_npy.mkdir(parents=True, exist_ok=True) save_dir_gt_npy.mkdir(parents=True, exist_ok=True) np.save( Path(save_dir_in_npy, filename).with_suffix(".npy"), multimodal) np.save( Path(save_dir_gt_npy, filename).with_suffix(".npy"), to_save[-1]) save_dir_distmap_npy.mkdir(parents=True, exist_ok=True) np.save( Path(save_dir_distmap_npy, filename).with_suffix(".npy"), dist_s) return neg, pos, space_dict, (rx, ry, rz)