def data2files(data_dir, cube_shape, step, shape=(50, 50), resize_labels_fac=1): data_names = [] for (rootDir, dirNames, filenames) in os.walk(data_dir): for filename in filenames: if filename[-5:] == '.pklz': data_names.append(filename) for i, dirname in enumerate(data_names): print 'Processing dir #%i/%i, %s:' % (i + 1, len(data_names), dirname), data_path = os.path.join(data_dir, dirname) print 'slicing ...', data_cubes, label_cubes = slice_data(data_path, reshape=False, cube_shape=cube_shape, step=step) print 'saving to list ...', # data po rezech do listu data_cubes_resh = [] for d in data_cubes: # print d.shape for s in d: data_cubes_resh.append(misc.resize_to_shape(s, shape=shape)) # labely po rezech do listu label_cubes_resh = [] for d in label_cubes: # print d.shape for s in d: new_shape = tuple((x / resize_labels_fac for x in shape)) lab = misc.resize_to_shape(s, shape=new_shape) label_cubes_resh.append(lab) posit_flag = np.array(label_cubes_resh).sum(axis=1).sum(axis=1) > 0 perc_posit = 100 * (posit_flag).sum() / len(posit_flag) print 'rel. posit. = %.1f%% ... ' % perc_posit, print 'done' # data na disk res_dir = '/home/tomas/Data/medical/dataset/gt/slicewise' if not os.path.exists(res_dir): os.mkdir(res_dir) print 'Saving data file ...', data_fname = os.path.join(res_dir, 'data.npz') data_file = gzip.open(data_fname, 'wb', compresslevel=1) np.save(data_file, np.array(data_cubes_resh)) data_file.close() print 'done' print 'Saving label file ...', label_fname = os.path.join(res_dir, 'labels.npz') label_file = gzip.open(label_fname, 'wb', compresslevel=1) np.save(label_file, np.array(label_cubes_resh)) label_file.close() print 'done'
def slice_data(fname, reshape=True, cube_shape=(20, 20, 20), step=10): datap = tools.load_pickle_data(fname, return_datap=True) data3d = datap['data3d'] labels = datap['segmentation'] voxelsize_mm = datap['voxelsize_mm'] data_cubes = [] label_cubes = [] # data3d = np.arange(1, 401).reshape((4, 10, 10)) # labels = 2 * np.ones((4, 10, 10)) # resizing data to have same spacing if reshape: orig_shape = data3d.shape n_slices = orig_shape[0] * (voxelsize_mm[0] / voxelsize_mm[1]) new_shape = (n_slices, orig_shape[1], orig_shape[2]) data3d = misc.resize_to_shape(data3d, shape=new_shape, order=1) labels = misc.resize_to_shape(labels, shape=new_shape, order=0) # print [(x, (labels == x).sum()) for x in np.unique(labels)] lesion_lbl = 1 liver_lbl = 2 bgd_lbl = 0 data_cubes_all = [ cube[4] for cube in tools.sliding_window_3d( data3d, step, cube_shape, only_whole=True, include_last=True) ] label_cubes_all = [ cube[4] for cube in tools.sliding_window_3d( labels, step, cube_shape, only_whole=True, include_last=True) ] areaT = 0.5 * np.prod( cube_shape) # prah pro minimalni zastoupeni jater v kostce # for data_cube, label_cube in zip(tools.sliding_window_3d(data3d, 10, cube_shape, only_whole=False, include_last=True)[4], # tools.sliding_window_3d(labels, 10, cube_shape, only_whole=False, include_last=True)[4]): for data_cube, label_cube in zip(data_cubes_all, label_cubes_all): if (label_cube == liver_lbl).sum() >= areaT: mean_v = data_cube[np.nonzero(label_cube == liver_lbl)].mean() # tmp = data_cube[0,...].copy() data_cube = np.where(label_cube == bgd_lbl, mean_v, data_cube) # plt.figure() # plt.subplot(131), plt.imshow(tmp, 'gray', vmin=-125, vmax=225), plt.title('orig') # plt.subplot(132), plt.imshow(data_cube[0,...], 'gray', vmin=-125, vmax=225), plt.title('processed, mean=%.1f' % mean_v) # plt.subplot(133), plt.imshow(label_cube[0,...], 'gray', vmin=0, vmax=2), plt.title('mask') # plt.show() data_cubes.append(data_cube) label_cubes.append( label_cube == lesion_lbl) # ulozim pouze oznacena loziska # seg_viewer.show(data3d, 100 * labels) return data_cubes, label_cubes
def get_lungs(self): lungs = scipy.ndimage.filters.gaussian_filter(self.data3dr, sigma=[4, 2, 2]) > -150 lungs[0, :, :] = 1 lungs = scipy.ndimage.morphology.binary_fill_holes(lungs) labs, n = scipy.ndimage.measurements.label(lungs==0) cornerlab = [ labs[0,0,0], labs[0,0,-1], labs[0,-1,0], labs[0,-1,-1], labs[-1,0,0], labs[-1,0,-1], labs[-1,-1,0], labs[-1,-1,-1] ] lb = np.median(cornerlab) labs[labs==lb] = 0 labs[labs==labs[0,0,0]] = 0 labs[labs==labs[0,0,-1]] = 0 labs[labs==labs[0,-1,0]] = 0 labs[labs==labs[0,-1,-1]] = 0 labs[labs==labs[-1,0,0]] = 0 labs[labs==labs[-1,0,-1]] = 0 labs[labs==labs[-1,-1,0]] = 0 labs[labs==labs[-1,-1,-1]] = 0 lungs = labs > 0 self.lungs = lungs #self.body = (labs == 80) return misc.resize_to_shape(lungs, self.orig_shape)
def dist_axial(self): if self.diaphragm_mask is None: self.get_diaphragm_mask() axdst = np.ones(self.data3dr.shape, dtype=np.int16) axdst[0 ,: ,:] = 0 iz, ix, iy = np.nonzero(self.diaphragm_mask) # print 'dia level ', self.diaphragm_mask_level axdst = scipy.ndimage.morphology.distance_transform_edt(axdst) - int(self.diaphragm_mask_level) return misc.resize_to_shape(axdst, self.orig_shape)
def differencing(path, res, axis): sez = list() teacher = list() dr = io3d.DataReader() datap = dr.Get3DData(path, dataplus_format=True) ed = sed3.sed3(datap['data3d']) datap['data3d'] = misc.resize_to_shape(datap['data3d'], [datap['data3d'].shape[0],100,100]) ed.show() try: under = np.where(ed.seeds == 1)[0][0] except: under = None try: over = np.where(ed.seeds == 3)[0][0] except: over = None number_of_slice = np.shape(ed.seeds)[0] c = list() if under is None and over is None: return [],[] elif under is None or over is None: print u'byla zadana jenom jedna mez zadejte obe nebo zadnou' w, y = differencing(path, res) return w, y if over > under: c = [1 for i in range(under)] b = [2 for i in range(over-under)] c.extend(b) b = [3 for i in range(number_of_slice-over)] c.extend(b) else: c = [3 for i in range(over)] b = [2 for i in range(under - over)] c.extend(b) b = [1 for i in range(number_of_slice - under)] c.extend(b) over, under = under, over for i in range(number_of_slice): a = datap['data3d'][i,:,:] x = c[i] teacher.append(x) sez.append(a) return sez, teacher, under, over
def get_diaphragm_mask(self, axis=0): if self.lungs is None: self.get_lungs() ia, ib, ic = self._get_ia_ib_ic(axis) data = self.lungs ou = self.get_diaphragm_profile_image(axis=axis) # reconstruction mask array mask = np.zeros(data.shape) for i in range(mask.shape[ia]): if ia == 0: mask[i,:,:] = ou > i elif ia == 1: mask[:,i,:] = ou > i elif ia == 2: mask[:,:,i] = ou > i self.diaphragm_mask = mask # maximal point is used for axial ze # ro plane self.diaphragm_mask_level = np.median(ou) self.center0 = self.diaphragm_mask_level * self.working_vs[0] return misc.resize_to_shape(self.diaphragm_mask, self.orig_shape)
def dist_to_surface(self): if self.body is None: self.get_body() ld = scipy.ndimage.morphology.distance_transform_edt(self.body) return misc.resize_to_shape(ld, self.orig_shape)
def dist_to_spine(self): if self.spine is None: self.get_spine() ld = scipy.ndimage.morphology.distance_transform_edt(1 - self.spine) return misc.resize_to_shape(ld, self.orig_shape)
def sliver_preparation(datadirpath, output_datadirpath="output_data", res=100, ax=0, organ='liver'): csvpath = output_datadirpath + '/sliver_label_' + str(res) + '_' + str( ax) + '_' + organ + '.csv' stat = output_datadirpath + '/sliver_stat' + str(res) + '_' + str( ax) + '_' + organ + '.csv' # datadirpath = '/home/trineon/projects/metalisa/data/SLIVER' f = h5py.File( output_datadirpath + '/sliver_' + str(res) + '_' + str(ax) + '_' + organ + '.hdf5', 'a') num = 1 for image in glob.glob(datadirpath + '/*orig*.mhd'): group = f.create_group(image.split('/')[-1]) orig, _ = DR.read(image) if ax != 0: orig = np.rollaxis(orig, ax) i = orig.shape[0] orig = misc.resize_to_shape(orig, [i, res, res]) DW.write(orig, output_datadirpath + '/sliver_' + str(num) + '_' + str(res) + '_' + str(ax) + '.pklz', metadata={"voxelsize_mm": [1, 1, 1]}) filename = output_datadirpath + '/sliver' + str(num) + '_' + str( res) + '_' + str(ax) + '.pklz' num += 1 seg = image.replace('orig', 'seg') lab, _ = DR.read(seg) if ax != 0: lab = np.rollaxis(lab, ax) l = list() a = 1 for slice in lab: if len(np.unique(slice)) > 1: l.append(2) a = 2 else: if a == 2: l.append(3) else: l.append(1) del lab for ind, slice in enumerate(orig): name = str(ind) dset = group.create_dataset(name, data=slice) dset.attrs['teacher'] = l[ind] dset.attrs['origin file'] = filename if l[-1] == 2: x = len(l) else: x = l.index(3) if l[0] == 2: y = 0 else: y = l.index(2) dt = { 'filename': [filename, filename, filename], 'label': ['under ' + organ, organ, 'above ' + organ], 'start_slice_number': [0, y, x], 'stop_slice_number': [y - 1, x - 1, len(l) - 1], 'axis': ax } if dt['stop_slice_number'][0] == -1: dt['stop_slice_number'][0] = 0 if os.path.exists(csvpath): new_df = pd.read_csv(csvpath) df = pd.DataFrame.from_dict(dt) new_df = pd.concat([new_df, df], ignore_index=True) else: df0 = pd.DataFrame.from_dict(dt) new_df = df0 new_df.to_csv(csvpath, index=False) a = y b = x - y c = len(l) - x dt = { 'filename': [filename], 'under liver': [a], 'liver': [b], 'above liver': [c], 'slices': [len(l)] } if os.path.exists(stat): new_df = pd.read_csv(stat) df = pd.DataFrame.from_dict(dt) new_df = pd.concat([new_df, df], ignore_index=True) else: df0 = pd.DataFrame.from_dict(dt) new_df = df0 new_df.to_csv(stat, index=False) pass
def ircad_preparation(datadirpath, output_datadirpath="output_data", organ="liver", res=100, ax=0): #test stat = output_datadirpath + '/stat_ircad' + str(res) + '_' + str( ax) + '_' + organ + '.csv' csvpath = output_datadirpath + '/label_ircad_' + str(res) + '_' + str( ax) + '_' + organ + '.csv' # datadirpath = '/home/trineon/projects/metalisa/data/IRCAD' seznam = [None] * 20 for folder in glob.glob(datadirpath + '/Pacient/*/'): count = len(glob.glob(folder + '*')) l = [None] * count for image in glob.glob(folder + '*'): number = int(image.split('/')[-1].split('_')[-1]) - 1 l[number], _ = DR.read(image) if ax != 0: l[number] = np.rollaxis(l[number], ax) for ind, i in enumerate(l): l[ind] = misc.resize_to_shape(i, [1, res, res]) scan = np.array(l) if ax != 0: np.rollaxis(scan, ax) name = folder.split('/')[-2] scan = scan.squeeze() DW.write(scan, output_datadirpath + '/IRCAD_' + str(name) + '_' + str(res) + '_' + str(ax) + '.pklz', metadata={"voxelsize_mm": [1, 1, 1]}) seznam[int(name) - 1] = output_datadirpath + '/IRCAD_' + str( name) + '_' + str(res) + '_' + str(ax) + '.pklz' ll = [None] * 20 for folder in glob.glob(datadirpath + '/labels/*/' + organ + '/'): count = len(glob.glob(folder + '*')) sez = list() for image in glob.glob(folder + 'IRCAD*.pklz'): label, _ = DR.read(image) if ax != 0: label = np.rollaxis(label, ax) l = list() a = 1 for slice in label: if len(np.unique(slice)) > 1: l.append(2) a = 2 else: if a == 2: l.append(3) else: l.append(1) ll[int(folder.split('/')[-3]) - 1] = l file = seznam[int(folder.split('/')[-3]) - 1] if l[-1] == 2: x = len(l) else: x = l.index(3) if l[0] == 2: y = 0 else: y = l.index(2) dt = { 'filename': [file, file, file], 'label': ['under ' + organ, organ, 'above ' + organ], 'start_slice_number': [0, y, x], 'stop_slice_number': [y - 1, x - 1, len(l) - 1], 'axis': ax } if dt['stop_slice_number'][0] == -1: dt['stop_slice_number'][0] = 0 if os.path.exists(csvpath): new_df = pd.read_csv(csvpath) df = pd.DataFrame.from_dict(dt) new_df = pd.concat([new_df, df], ignore_index=True) else: df0 = pd.DataFrame.from_dict(dt) new_df = df0 new_df.to_csv(csvpath, index=False) a = y b = x - y c = len(l) - x dt = { 'filename': [file], 'under liver': [a], 'liver': [b], 'above liver': [c], 'slices': [len(l)] } if os.path.exists(stat): new_df = pd.read_csv(stat) df = pd.DataFrame.from_dict(dt) new_df = pd.concat([new_df, df], ignore_index=True) else: df0 = pd.DataFrame.from_dict(dt) new_df = df0 new_df.to_csv(stat, index=False) output_datadirpath + '/sliver_' + str(res) + '_' + str( ax) + '_' + organ + '.hdf5' f = h5py.File( output_datadirpath + '/sliver_' + str(res) + '_' + str(ax) + '_' + str(organ) + '.hdf5', 'a') for index, file in enumerate(seznam): group = f.create_group(file.split('/')[-1]) l = ll[index] scan, _ = DR.read(file) for ind, slice in enumerate(scan): name = str(ind) dset = group.create_dataset(name, data=slice) dset.attrs['teacher'] = l[ind] dset.attrs['origin file'] = file f.close() pass