def convert_input(inputFolder, saveLocation, remove_bad=True):
"""Function for converting data from imageJ ROIs + data to mem_mapped arrays for preprocessing + batch generation to pass to cnn
"""
#get pairs
tfs = listdirfull(inputFolder, keyword=".tif")
zps = [xx for xx in listdirfull(inputFolder) if ".tif" not in xx]
#make empty zip files if no labels (useful to train on negative data?)
for tf in tfs:
if tf[:-4] + "RoiSet.zip" not in zps:
print(tf)
nm = tf[:-4] + "RoiSet.zip"
with zipfile.ZipFile(os.path.join(inputFolder, nm), "w") as file:
pass
pairs = [[tf, zp] for tf in tfs for zp in zps if tf[:-4] in zp]
#make saveLocation if doesn"t exist:
makedir(saveLocation)
#make mem_mapped arrays once, to be more cluster friendly
print("Starting conversion...")
p = mp.Pool(12)
iterlst = [(pair[0], pair[1], saveLocation) for pair in pairs]
bad = p.starmap(basic_convert, iterlst)
p.terminate()
print("Completed!\n\nBad list: {}".format(bad))
#check all and make dictionary of points (nx3)
file_points_dct = {}
print("Checking all files have valid input and anns...")
for a in listdirfull(saveLocation, "npy"):
arr = load_np(a)
sh = np.nonzero(arr[0])[0].shape[0]
pnts = np.nonzero(arr[1])
shh = pnts[0].shape[0]
if sh == 0 or shh == 0:
print(
"File: {} \n input images nonzeros=={}, annotation nonzeros=={}"
.format(a, sh, shh))
if remove_bad:
os.remove(a)
print("removing")
else:
file_points_dct[os.path.basename(a)] = list(zip(*pnts))
#save out points
save_dictionary(
os.path.join(os.path.dirname(saveLocation), "points_dictionary.p"),
file_points_dct)
print("Saved dictionary in {}".format(saveLocation))
return
def otsu_par(saveLocation, otsufld, size, otsu_factor):
"""
for inn in listdirfull(saveLocation, "npy"):
print inn
#load
arr = load_np(inn)
raw = np.copy(arr[0])
lbl = np.copy(arr[1])
#save input
inrawfld = os.path.join(otsufld, "inputRawImages"); makedir(inrawfld)
#[tifffile.imsave(os.path.join(inrawfld, "{}_{}_inputRawImages.tif".format(os.path.basename(inn)[:-4], str(i).zfill(4))), ii) for i, ii in enumerate(arr[0])]
f = h5.File(os.path.join(inrawfld, "{}_inputRawImages.h5".format(os.path.basename(inn)[:-4])), "w")
f["/main"] = raw
f.close()
#save input
#OTSU?
inlblfld = os.path.join(otsufld, "inputLabelImages"); makedir(inlblfld)
otsu = otsu_dilate(raw, lbl, size=size, otsu_factor=otsu_factor).astype("float32")
f = h5.File(os.path.join(inlblfld, "{}_inputLabelImages-segmentation.h5".format(os.path.basename(inn)[:-4])), "w")
f["/main"] = otsu
f.close()
"""
#otsu
p = mp.Pool(12)
iterlst = [(otsufld, inn, size, otsu_factor) for inn in listdirfull(saveLocation, "npy")]
p.starmap(otsu_helper, iterlst)
p.terminate()
return
def otsu_par(saveLocation, otsufld, size, otsu_factor):
#otsu
p = mp.Pool(12)
iterlst = [(otsufld, inn, size, otsu_factor)
for inn in listdirfull(saveLocation, "npy")]
p.starmap(otsu_helper, iterlst)
p.terminate()
return
if verbose:
sys.stdout.write(
"done.\n\n***Memmapped array generated successfully***\n\n")
sys.stdout.flush()
return arr
#%%
if __name__ == "__main__":
#convert first
inputFolder = "/home/wanglab/Documents/cfos_raw_inputs/"
saveLocation = "/home/wanglab/Documents/cfos_inputs/memmap"
makedir(saveLocation)
otsufld = "/home/wanglab/Documents/cfos_inputs/otsu"
makedir(otsufld)
size = (5, 10, 10)
otsu_factor = 0.8
#convert
convert_input(inputFolder, saveLocation, remove_bad=True)
#check all
for a in listdirfull(saveLocation, "npy"):
sh = np.nonzero(load_np(a))[0].shape[0]
if sh == 0: print(a, sh)
#%%
#otsu_par
otsu_par(saveLocation, otsufld, size, otsu_factor)