def convert_input(inputFolder, saveLocation, remove_bad=True):
"""Function for converting data from imageJ ROIs + data to mem_mapped arrays for preprocessing + batch generation to pass to cnn
"""
#get pairs
tfs = listdirfull(inputFolder, keyword=".tif")
zps = [xx for xx in listdirfull(inputFolder) if ".tif" not in xx]
#make empty zip files if no labels (useful to train on negative data?)
for tf in tfs:
if tf[:-4] + "RoiSet.zip" not in zps:
print(tf)
nm = tf[:-4] + "RoiSet.zip"
with zipfile.ZipFile(os.path.join(inputFolder, nm), "w") as file:
pass
pairs = [[tf, zp] for tf in tfs for zp in zps if tf[:-4] in zp]
#make saveLocation if doesn"t exist:
makedir(saveLocation)
#make mem_mapped arrays once, to be more cluster friendly
print("Starting conversion...")
p = mp.Pool(12)
iterlst = [(pair[0], pair[1], saveLocation) for pair in pairs]
bad = p.starmap(basic_convert, iterlst)
p.terminate()
print("Completed!\n\nBad list: {}".format(bad))
#check all and make dictionary of points (nx3)
file_points_dct = {}
print("Checking all files have valid input and anns...")
for a in listdirfull(saveLocation, "npy"):
arr = load_np(a)
sh = np.nonzero(arr[0])[0].shape[0]
pnts = np.nonzero(arr[1])
shh = pnts[0].shape[0]
if sh == 0 or shh == 0:
print(
"File: {} \n input images nonzeros=={}, annotation nonzeros=={}"
.format(a, sh, shh))
if remove_bad:
os.remove(a)
print("removing")
else:
file_points_dct[os.path.basename(a)] = list(zip(*pnts))
#save out points
save_dictionary(
os.path.join(os.path.dirname(saveLocation), "points_dictionary.p"),
file_points_dct)
print("Saved dictionary in {}".format(saveLocation))
return
def otsu_helper(otsufld, inn, size, otsu_factor):
#load
arr = load_np(inn)
raw = np.copy(arr[0])
lbl = np.copy(arr[1])
#save input
otsu = otsu_dilate(raw, lbl, size=size, otsu_factor=otsu_factor)
merged = np.stack([raw, otsu, np.zeros_like(raw)], -1)
tifffile.imsave(os.path.join(otsufld, "{}_otsu{}_size{}x{}x{}_lbl_overlay.tif".format(os.path.basename(inn)[:-4], otsu_factor,
size[0], size[1], size[2])), merged)
print(inn)
return
def otsu_helper(otsufld, inn, size, otsu_factor):
#load
arr = load_np(inn)
raw = np.copy(arr[0])
lbl = np.copy(arr[1])
#save input
tifffile.imsave(
os.path.join(otsufld, "{}_img.tif".format(os.path.basename(inn)[:-4])),
raw.astype("float32"))
#save input
otsu = otsu_dilate(raw, lbl, size=size, otsu_factor=otsu_factor)
tifffile.imsave(
os.path.join(otsufld, "{}_lbl.tif".format(os.path.basename(inn)[:-4])),
otsu.astype("float32"))
print(inn)
return
if verbose:
sys.stdout.write(
"done.\n\n***Memmapped array generated successfully***\n\n")
sys.stdout.flush()
return arr
#%%
if __name__ == "__main__":
#convert first
inputFolder = "/home/wanglab/Documents/cfos_raw_inputs/"
saveLocation = "/home/wanglab/Documents/cfos_inputs/memmap"
makedir(saveLocation)
otsufld = "/home/wanglab/Documents/cfos_inputs/otsu"
makedir(otsufld)
size = (5, 10, 10)
otsu_factor = 0.8
#convert
convert_input(inputFolder, saveLocation, remove_bad=True)
#check all
for a in listdirfull(saveLocation, "npy"):
sh = np.nonzero(load_np(a))[0].shape[0]
if sh == 0: print(a, sh)
#%%
#otsu_par
otsu_par(saveLocation, otsufld, size, otsu_factor)