def data2files(data_dir,
cube_shape,
step,
shape=(50, 50),
resize_labels_fac=1):
data_names = []
for (rootDir, dirNames, filenames) in os.walk(data_dir):
for filename in filenames:
if filename[-5:] == '.pklz':
data_names.append(filename)
for i, dirname in enumerate(data_names):
print 'Processing dir #%i/%i, %s:' % (i + 1, len(data_names), dirname),
data_path = os.path.join(data_dir, dirname)
print 'slicing ...',
data_cubes, label_cubes = slice_data(data_path,
reshape=False,
cube_shape=cube_shape,
step=step)
print 'saving to list ...',
# data po rezech do listu
data_cubes_resh = []
for d in data_cubes:
# print d.shape
for s in d:
data_cubes_resh.append(misc.resize_to_shape(s, shape=shape))
# labely po rezech do listu
label_cubes_resh = []
for d in label_cubes:
# print d.shape
for s in d:
new_shape = tuple((x / resize_labels_fac for x in shape))
lab = misc.resize_to_shape(s, shape=new_shape)
label_cubes_resh.append(lab)
posit_flag = np.array(label_cubes_resh).sum(axis=1).sum(axis=1) > 0
perc_posit = 100 * (posit_flag).sum() / len(posit_flag)
print 'rel. posit. = %.1f%% ... ' % perc_posit,
print 'done'
# data na disk
res_dir = '/home/tomas/Data/medical/dataset/gt/slicewise'
if not os.path.exists(res_dir):
os.mkdir(res_dir)
print 'Saving data file ...',
data_fname = os.path.join(res_dir, 'data.npz')
data_file = gzip.open(data_fname, 'wb', compresslevel=1)
np.save(data_file, np.array(data_cubes_resh))
data_file.close()
print 'done'
print 'Saving label file ...',
label_fname = os.path.join(res_dir, 'labels.npz')
label_file = gzip.open(label_fname, 'wb', compresslevel=1)
np.save(label_file, np.array(label_cubes_resh))
label_file.close()
print 'done'
def slice_data(fname, reshape=True, cube_shape=(20, 20, 20), step=10):
datap = tools.load_pickle_data(fname, return_datap=True)
data3d = datap['data3d']
labels = datap['segmentation']
voxelsize_mm = datap['voxelsize_mm']
data_cubes = []
label_cubes = []
# data3d = np.arange(1, 401).reshape((4, 10, 10))
# labels = 2 * np.ones((4, 10, 10))
# resizing data to have same spacing
if reshape:
orig_shape = data3d.shape
n_slices = orig_shape[0] * (voxelsize_mm[0] / voxelsize_mm[1])
new_shape = (n_slices, orig_shape[1], orig_shape[2])
data3d = misc.resize_to_shape(data3d, shape=new_shape, order=1)
labels = misc.resize_to_shape(labels, shape=new_shape, order=0)
# print [(x, (labels == x).sum()) for x in np.unique(labels)]
lesion_lbl = 1
liver_lbl = 2
bgd_lbl = 0
data_cubes_all = [
cube[4] for cube in tools.sliding_window_3d(
data3d, step, cube_shape, only_whole=True, include_last=True)
]
label_cubes_all = [
cube[4] for cube in tools.sliding_window_3d(
labels, step, cube_shape, only_whole=True, include_last=True)
]
areaT = 0.5 * np.prod(
cube_shape) # prah pro minimalni zastoupeni jater v kostce
# for data_cube, label_cube in zip(tools.sliding_window_3d(data3d, 10, cube_shape, only_whole=False, include_last=True)[4],
# tools.sliding_window_3d(labels, 10, cube_shape, only_whole=False, include_last=True)[4]):
for data_cube, label_cube in zip(data_cubes_all, label_cubes_all):
if (label_cube == liver_lbl).sum() >= areaT:
mean_v = data_cube[np.nonzero(label_cube == liver_lbl)].mean()
# tmp = data_cube[0,...].copy()
data_cube = np.where(label_cube == bgd_lbl, mean_v, data_cube)
# plt.figure()
# plt.subplot(131), plt.imshow(tmp, 'gray', vmin=-125, vmax=225), plt.title('orig')
# plt.subplot(132), plt.imshow(data_cube[0,...], 'gray', vmin=-125, vmax=225), plt.title('processed, mean=%.1f' % mean_v)
# plt.subplot(133), plt.imshow(label_cube[0,...], 'gray', vmin=0, vmax=2), plt.title('mask')
# plt.show()
data_cubes.append(data_cube)
label_cubes.append(
label_cube == lesion_lbl) # ulozim pouze oznacena loziska
# seg_viewer.show(data3d, 100 * labels)
return data_cubes, label_cubes
def get_lungs(self):
lungs = scipy.ndimage.filters.gaussian_filter(self.data3dr, sigma=[4, 2, 2]) > -150
lungs[0, :, :] = 1
lungs = scipy.ndimage.morphology.binary_fill_holes(lungs)
labs, n = scipy.ndimage.measurements.label(lungs==0)
cornerlab = [
labs[0,0,0],
labs[0,0,-1],
labs[0,-1,0],
labs[0,-1,-1],
labs[-1,0,0],
labs[-1,0,-1],
labs[-1,-1,0],
labs[-1,-1,-1]
]
lb = np.median(cornerlab)
labs[labs==lb] = 0
labs[labs==labs[0,0,0]] = 0
labs[labs==labs[0,0,-1]] = 0
labs[labs==labs[0,-1,0]] = 0
labs[labs==labs[0,-1,-1]] = 0
labs[labs==labs[-1,0,0]] = 0
labs[labs==labs[-1,0,-1]] = 0
labs[labs==labs[-1,-1,0]] = 0
labs[labs==labs[-1,-1,-1]] = 0
lungs = labs > 0
self.lungs = lungs
#self.body = (labs == 80)
return misc.resize_to_shape(lungs, self.orig_shape)
def dist_axial(self):
if self.diaphragm_mask is None:
self.get_diaphragm_mask()
axdst = np.ones(self.data3dr.shape, dtype=np.int16)
axdst[0 ,: ,:] = 0
iz, ix, iy = np.nonzero(self.diaphragm_mask)
# print 'dia level ', self.diaphragm_mask_level
axdst = scipy.ndimage.morphology.distance_transform_edt(axdst) - int(self.diaphragm_mask_level)
return misc.resize_to_shape(axdst, self.orig_shape)
def differencing(path, res, axis):
sez = list()
teacher = list()
dr = io3d.DataReader()
datap = dr.Get3DData(path, dataplus_format=True)
ed = sed3.sed3(datap['data3d'])
datap['data3d'] = misc.resize_to_shape(datap['data3d'], [datap['data3d'].shape[0],100,100])
ed.show()
try:
under = np.where(ed.seeds == 1)[0][0]
except:
under = None
try:
over = np.where(ed.seeds == 3)[0][0]
except:
over = None
number_of_slice = np.shape(ed.seeds)[0]
c = list()
if under is None and over is None:
return [],[]
elif under is None or over is None:
print u'byla zadana jenom jedna mez zadejte obe nebo zadnou'
w, y = differencing(path, res)
return w, y
if over > under:
c = [1 for i in range(under)]
b = [2 for i in range(over-under)]
c.extend(b)
b = [3 for i in range(number_of_slice-over)]
c.extend(b)
else:
c = [3 for i in range(over)]
b = [2 for i in range(under - over)]
c.extend(b)
b = [1 for i in range(number_of_slice - under)]
c.extend(b)
over, under = under, over
for i in range(number_of_slice):
a = datap['data3d'][i,:,:]
x = c[i]
teacher.append(x)
sez.append(a)
return sez, teacher, under, over
def get_diaphragm_mask(self, axis=0):
if self.lungs is None:
self.get_lungs()
ia, ib, ic = self._get_ia_ib_ic(axis)
data = self.lungs
ou = self.get_diaphragm_profile_image(axis=axis)
# reconstruction mask array
mask = np.zeros(data.shape)
for i in range(mask.shape[ia]):
if ia == 0:
mask[i,:,:] = ou > i
elif ia == 1:
mask[:,i,:] = ou > i
elif ia == 2:
mask[:,:,i] = ou > i
self.diaphragm_mask = mask
# maximal point is used for axial ze
# ro plane
self.diaphragm_mask_level = np.median(ou)
self.center0 = self.diaphragm_mask_level * self.working_vs[0]
return misc.resize_to_shape(self.diaphragm_mask, self.orig_shape)
def dist_to_surface(self):
if self.body is None:
self.get_body()
ld = scipy.ndimage.morphology.distance_transform_edt(self.body)
return misc.resize_to_shape(ld, self.orig_shape)
def dist_to_spine(self):
if self.spine is None:
self.get_spine()
ld = scipy.ndimage.morphology.distance_transform_edt(1 - self.spine)
return misc.resize_to_shape(ld, self.orig_shape)
def sliver_preparation(datadirpath,
output_datadirpath="output_data",
res=100,
ax=0,
organ='liver'):
csvpath = output_datadirpath + '/sliver_label_' + str(res) + '_' + str(
ax) + '_' + organ + '.csv'
stat = output_datadirpath + '/sliver_stat' + str(res) + '_' + str(
ax) + '_' + organ + '.csv'
# datadirpath = '/home/trineon/projects/metalisa/data/SLIVER'
f = h5py.File(
output_datadirpath + '/sliver_' + str(res) + '_' + str(ax) + '_' +
organ + '.hdf5', 'a')
num = 1
for image in glob.glob(datadirpath + '/*orig*.mhd'):
group = f.create_group(image.split('/')[-1])
orig, _ = DR.read(image)
if ax != 0:
orig = np.rollaxis(orig, ax)
i = orig.shape[0]
orig = misc.resize_to_shape(orig, [i, res, res])
DW.write(orig,
output_datadirpath + '/sliver_' + str(num) + '_' + str(res) +
'_' + str(ax) + '.pklz',
metadata={"voxelsize_mm": [1, 1, 1]})
filename = output_datadirpath + '/sliver' + str(num) + '_' + str(
res) + '_' + str(ax) + '.pklz'
num += 1
seg = image.replace('orig', 'seg')
lab, _ = DR.read(seg)
if ax != 0:
lab = np.rollaxis(lab, ax)
l = list()
a = 1
for slice in lab:
if len(np.unique(slice)) > 1:
l.append(2)
a = 2
else:
if a == 2:
l.append(3)
else:
l.append(1)
del lab
for ind, slice in enumerate(orig):
name = str(ind)
dset = group.create_dataset(name, data=slice)
dset.attrs['teacher'] = l[ind]
dset.attrs['origin file'] = filename
if l[-1] == 2:
x = len(l)
else:
x = l.index(3)
if l[0] == 2:
y = 0
else:
y = l.index(2)
dt = {
'filename': [filename, filename, filename],
'label': ['under ' + organ, organ, 'above ' + organ],
'start_slice_number': [0, y, x],
'stop_slice_number': [y - 1, x - 1, len(l) - 1],
'axis': ax
}
if dt['stop_slice_number'][0] == -1:
dt['stop_slice_number'][0] = 0
if os.path.exists(csvpath):
new_df = pd.read_csv(csvpath)
df = pd.DataFrame.from_dict(dt)
new_df = pd.concat([new_df, df], ignore_index=True)
else:
df0 = pd.DataFrame.from_dict(dt)
new_df = df0
new_df.to_csv(csvpath, index=False)
a = y
b = x - y
c = len(l) - x
dt = {
'filename': [filename],
'under liver': [a],
'liver': [b],
'above liver': [c],
'slices': [len(l)]
}
if os.path.exists(stat):
new_df = pd.read_csv(stat)
df = pd.DataFrame.from_dict(dt)
new_df = pd.concat([new_df, df], ignore_index=True)
else:
df0 = pd.DataFrame.from_dict(dt)
new_df = df0
new_df.to_csv(stat, index=False)
pass
def ircad_preparation(datadirpath,
output_datadirpath="output_data",
organ="liver",
res=100,
ax=0):
#test
stat = output_datadirpath + '/stat_ircad' + str(res) + '_' + str(
ax) + '_' + organ + '.csv'
csvpath = output_datadirpath + '/label_ircad_' + str(res) + '_' + str(
ax) + '_' + organ + '.csv'
# datadirpath = '/home/trineon/projects/metalisa/data/IRCAD'
seznam = [None] * 20
for folder in glob.glob(datadirpath + '/Pacient/*/'):
count = len(glob.glob(folder + '*'))
l = [None] * count
for image in glob.glob(folder + '*'):
number = int(image.split('/')[-1].split('_')[-1]) - 1
l[number], _ = DR.read(image)
if ax != 0:
l[number] = np.rollaxis(l[number], ax)
for ind, i in enumerate(l):
l[ind] = misc.resize_to_shape(i, [1, res, res])
scan = np.array(l)
if ax != 0:
np.rollaxis(scan, ax)
name = folder.split('/')[-2]
scan = scan.squeeze()
DW.write(scan,
output_datadirpath + '/IRCAD_' + str(name) + '_' + str(res) +
'_' + str(ax) + '.pklz',
metadata={"voxelsize_mm": [1, 1, 1]})
seznam[int(name) - 1] = output_datadirpath + '/IRCAD_' + str(
name) + '_' + str(res) + '_' + str(ax) + '.pklz'
ll = [None] * 20
for folder in glob.glob(datadirpath + '/labels/*/' + organ + '/'):
count = len(glob.glob(folder + '*'))
sez = list()
for image in glob.glob(folder + 'IRCAD*.pklz'):
label, _ = DR.read(image)
if ax != 0:
label = np.rollaxis(label, ax)
l = list()
a = 1
for slice in label:
if len(np.unique(slice)) > 1:
l.append(2)
a = 2
else:
if a == 2:
l.append(3)
else:
l.append(1)
ll[int(folder.split('/')[-3]) - 1] = l
file = seznam[int(folder.split('/')[-3]) - 1]
if l[-1] == 2:
x = len(l)
else:
x = l.index(3)
if l[0] == 2:
y = 0
else:
y = l.index(2)
dt = {
'filename': [file, file, file],
'label': ['under ' + organ, organ, 'above ' + organ],
'start_slice_number': [0, y, x],
'stop_slice_number': [y - 1, x - 1, len(l) - 1],
'axis': ax
}
if dt['stop_slice_number'][0] == -1:
dt['stop_slice_number'][0] = 0
if os.path.exists(csvpath):
new_df = pd.read_csv(csvpath)
df = pd.DataFrame.from_dict(dt)
new_df = pd.concat([new_df, df], ignore_index=True)
else:
df0 = pd.DataFrame.from_dict(dt)
new_df = df0
new_df.to_csv(csvpath, index=False)
a = y
b = x - y
c = len(l) - x
dt = {
'filename': [file],
'under liver': [a],
'liver': [b],
'above liver': [c],
'slices': [len(l)]
}
if os.path.exists(stat):
new_df = pd.read_csv(stat)
df = pd.DataFrame.from_dict(dt)
new_df = pd.concat([new_df, df], ignore_index=True)
else:
df0 = pd.DataFrame.from_dict(dt)
new_df = df0
new_df.to_csv(stat, index=False)
output_datadirpath + '/sliver_' + str(res) + '_' + str(
ax) + '_' + organ + '.hdf5'
f = h5py.File(
output_datadirpath + '/sliver_' + str(res) + '_' + str(ax) + '_' +
str(organ) + '.hdf5', 'a')
for index, file in enumerate(seznam):
group = f.create_group(file.split('/')[-1])
l = ll[index]
scan, _ = DR.read(file)
for ind, slice in enumerate(scan):
name = str(ind)
dset = group.create_dataset(name, data=slice)
dset.attrs['teacher'] = l[ind]
dset.attrs['origin file'] = file
f.close()
pass
