def process_file(FQ_file,
img_size=(960, 960),
bin_prop=(0, 90, 20),
channels={'cells': 'C3-'},
data_category={'roi': ''},
annotation_extension='__RoiSet.zip',
img_extension='.tif',
show_plot=None,
Zrange=None,
dZ=2):
'''
Function uses annotations generated in FIJI and creates mask based
on the specified parameters. The resulting files are zipped and be
used for training of a neural network with ImJoy.
Args:
Zrange [tuple, 2 elements]. [Optional] Tuple specifying minimum and maximum z-value that is considered
in analysis.
bin_prop [Tuple, 3 elements]. Specifies the bins for the histograms (min, max,delta).
'''
# Get input args. Has to be FIRST call!
input_args = locals()
# Make sure input args are correct - assignments with 0 can come from ImJoy
if Zrange[0] == 0 or Zrange[1] == 0:
Zrange = None
if bin_prop[1] == 0 or bin_prop[1] == 0:
bin_prop = (0, 90, 20)
## Prepare folder to save results
drive, path_and_file = os.path.splitdrive(FQ_file)
path_results, file_results = os.path.split(path_and_file)
file_base, ext = os.path.splitext(file_results)
path_save = os.path.join(
path_results, file_base,
'MembDist_{}'.format(time.strftime("%y%m%d-%H%M", time.localtime())))
if not os.path.isdir(path_save):
os.makedirs(path_save)
## Open FQ results
fq_dict = FQtoolbox.read_FQ_matlab(FQ_file)
spots_all = FQtoolbox.get_rna(fq_dict)
Zrna = spots_all[:, [18]]
# Open annotations
print('Open annotations')
if 'RoiSet.zip' in annotation_extension:
path_annot = os.path.join(path_results, 'zstack_segmentation')
folderImporter = annotationImporter.FolderImporter(
channels=channels,
data_category=data_category,
annot_ext=annotation_extension)
annotDict = folderImporter.load(path_annot)
print('average roi size:', annotDict['roi_size'])
# Generate binary masks for a selected data-set
binaryGen = maskGenerator.BinaryMaskGenerator(erose_size=5,
obj_size_rem=500,
save_indiv=True)
# The generate function uses as an input the sub-dictionary for one data-category and one channel
annotatFiles = annotDict['roi']['cells']
maskDict = binaryGen.generate(annotatFiles)
#Use a loop and the update function to add the mask dictionary to the loaded annotation dictonary\n",
for k, v in annotatFiles.items():
v.update(maskDict[k])
# Bins of histogram
binsHist = np.arange(bin_prop[0], bin_prop[1], bin_prop[2])
width = 0.8 * (binsHist[1] - binsHist[0])
center = (binsHist[:-1] + binsHist[1:]) / 2
# Other parameters for calculation
dist_membr_RNA = np.array([])
dist_membr_pix = np.array([])
idx = 0
# Loop over all z-slices
hist_slice = {}
print('Loop over slices')
for k, v in annotatFiles.items():
print(f'Slice: {k}')
# Get Z coordinate
m = re.search('.*_Z([0-9]*)\.tif', k)
Zmask = int(m.group(1))
# Check if outside of specified z range
if Zrange is not None:
if (Zmask <= Zrange[0]) or (Zmask >= Zrange[1]):
print('Slice outside of range')
continue
# Get z-range for loop
Zloop = np.logical_and(Zrna <= Zmask + dZ,
Zrna >= Zmask - dZ).flatten()
spots_loop = spots_all[Zloop, :]
spots_loop_XY = spots_loop[:, [16, 17]].astype(int)
# Distance transform
dist_membr = ndimage.distance_transform_edt(
~v['mask_edge']) # Negate mask
# Indices have to be inversed to access array
dist_membr_RNA_loop = dist_membr[spots_loop_XY[:, 0], spots_loop_XY[:,
1]]
# Get distance from membrane for all pixel in the cell
mask_all = v['mask_fill'] + v['mask_edge']
dist_membr_pix_loop = dist_membr[mask_all]
# Save values
if idx == 0:
dist_membr_RNA = np.copy(dist_membr_RNA_loop)
dist_membr_pix = np.copy(dist_membr_pix_loop)
else:
dist_membr_RNA = np.append(dist_membr_RNA,
dist_membr_RNA_loop,
axis=0)
dist_membr_pix = np.append(dist_membr_pix,
dist_membr_pix_loop,
axis=0)
idx += 1
# Calculate histograms
histRNA, bins = np.histogram(dist_membr_RNA_loop,
binsHist,
density=False)
histpix, bins = np.histogram(dist_membr_pix_loop,
binsHist,
density=False)
histRNAnorm = histRNA / histRNA.sum()
histpixnorm = histpix / histpix.sum()
histRNAnormPix = np.divide(histRNAnorm, histpixnorm)
histRNAnormPix = np.nan_to_num(histRNAnormPix)
hist_plot = {
'width': width,
'center': center,
'bins': bins,
'histRNA': histRNA,
'histpix': histpix,
'histRNAnormPix': histRNAnormPix
}
# Plot results
name_save = os.path.join(path_save, f'Z-{Zmask}.png')
plot_results_slice(Zmask, v, mask_all, spots_loop_XY, dist_membr,
hist_plot, name_save)
hist_slice[f'Z{Zmask}'] = hist_plot
# Analyze all slices
histRNA_all, bins = np.histogram(dist_membr_RNA, binsHist, density=False)
histRNA_all_norm = histRNA_all / histRNA_all.sum()
# Renormalize with pixel counts
histpix_all, bins = np.histogram(dist_membr_pix, binsHist, density=False)
histpix_all_norm = histpix_all / histpix_all.sum()
hist_RNA_all_normPix = np.divide(histRNA_all_norm, histpix_all_norm)
hist_RNA_all_normPix = np.nan_to_num(hist_RNA_all_normPix)
hist_plot_all = {
'width': width,
'center': center,
'bins': bins,
'histRNA_all': histRNA_all,
'histRNA_all_norm': histRNA_all_norm,
'histpix_all_norm': histpix_all_norm,
'hist_RNA_all_normPix': hist_RNA_all_normPix
}
name_save = os.path.join(path_save, '_DistanceEnrichmentSummary.png')
plot_results_all(hist_plot_all, name_save)
if show_plot:
show_plot(name_save)
# Save entire analysis results as json
input_args.pop('show_plot', None)
analysis_results = {
'args': input_args,
'hist_all': hist_plot_all,
'hist_slice': hist_slice
}
name_json = os.path.join(path_save, 'DataAll.json')
with open(name_json, 'w') as fp:
json.dump(analysis_results,
fp,
sort_keys=True,
indent=4,
cls=NumpyEncoder)
# Save histogram of pooled data as csv
name_csv = os.path.join(path_save, '_HistogramPooled.csv')
hist_plot_all.pop('bins', None)
hist_plot_all.pop('width', None)
csv_header = ';'.join(hist_plot_all.keys())
hist_values = np.array(list(hist_plot_all.values())).transpose()
np.savetxt(name_csv,
hist_values,
delimiter=";",
fmt='%f',
header=csv_header,
comments='')
return analysis_results
def calc_nuclear_enrichment(FQ_file, binsHist):
## Get folder
drive, path_and_file = os.path.splitdrive(FQ_file)
path_results, file_results = os.path.split(path_and_file)
# *************************************************************************
# Load nuclear outline
# Generate binary masks for a selected data-set
binaryGen = maskGenerator.BinaryMaskGenerator(erose_size=5,
obj_size_rem=500,
save_indiv=True,
progress_callback=None)
## Import annotations for nuclei
path_annot = os.path.join(drive, path_results, 'zstack_segmentation')
folderImporter = annotationImporter.FolderImporter(
channels={'nuclei': 'C4-'},
data_category={'roi': ''},
annot_ext='__RoiSet.zip',
progress_callback=None)
annotDict = folderImporter.load(path_annot)
print('average roi size:', annotDict['roi_size'])
# The generate function uses as an input the sub-dictionary for one data-category and one channel
annotatFiles = annotDict['roi']['nuclei']
mask_dict_nuclei = binaryGen.generate(annotatFiles)
keys_delete = []
for k, v in annotatFiles.items():
# Make sure that key in present in mask, otherwise delete
if k in mask_dict_nuclei:
v.update(mask_dict_nuclei[k])
else:
keys_delete.append(k)
# *************************************************************************
# Load embryo outline
file_embryo = os.path.join(drive, path_results, 'embryo_contour.roi')
# Conver dictionary & get size of ROIS
roi_import = read_roi_file(file_embryo)
roi_dict = {}
roi_dict['embryo'] = {}
roi_dict['embryo']['type'] = roi_import['embryo_contour']['type']
roi_dict['embryo']['pos'] = np.column_stack(
(roi_import['embryo_contour']['y'], roi_import['embryo_contour']['x']))
# Assemble structure to call mask generator
image_size = annotatFiles.values().__iter__().__next__()['image'].shape
image_fake = np.zeros(image_size, dtype=np.uint8)
annotat_files_embryo = {}
annotat_files_embryo['embryo_contour'] = {}
annotat_files_embryo['embryo_contour']['roi'] = roi_dict
annotat_files_embryo['embryo_contour']['image'] = image_fake
mask_dict_embryo = binaryGen.generate(annotat_files_embryo)
mask_embryo = mask_dict_embryo['embryo_contour']['mask_fill']
# *************************************************************************
# Load and analyze FQ results
fq_dict = FQtoolbox.read_FQ_matlab(file_load)
spots_all = FQtoolbox.get_rna(fq_dict)
# Z position in pixel
Zrna = np.divide(spots_all[:, [2]],
fq_dict['settings']['microscope']['pix_z']).astype(int)
# Other parameters for calculation
dist_membr_RNA = np.array([])
dist_membr_pix = np.array([])
idx = 0
# Loop over all z-slices
print(' == Loop over slices')
for idx_file, (k_annot, v_annot) in enumerate(annotatFiles.items()):
print(f'Slice: {k_annot}')
# Get Z coordinate
m = re.search('.*_Z([0-9]*)\.tif', k_annot)
Zmask = int(m.group(1))
# Check if outside of specified z range
if Zrange is not None:
if (Zmask < Zrange[0]) or (Zmask > Zrange[1]):
print(f'Z-slice outside of specified range: {Zmask}')
continue
# Get z-range for loop
Zloop = np.logical_and(Zrna <= Zmask + dZ,
Zrna >= Zmask - dZ).flatten()
Zloop = (Zrna == Zmask).flatten()
spots_loop = spots_all[Zloop, :]
spots_loop_XY = np.divide(
spots_loop[:, [0, 1]],
fq_dict['settings']['microscope']['pix_xy']).astype(int)
# Distance transform
dist_nuc_outside = ndimage.distance_transform_edt(
~v_annot['mask_fill'])
dist_nuc_inside = ndimage.distance_transform_edt(
v_annot['mask_fill']) # Negate mask
dist_nuc = dist_nuc_outside - dist_nuc_inside
# Indices have to be inversed to access array
dist_nuc_RNA_loop = dist_nuc[spots_loop_XY[:, 0], spots_loop_XY[:, 1]]
# Get distance from membrane for all pixel in the cell
dist_membr_pix_loop = dist_nuc[mask_embryo.astype(bool)]
# Save values
if idx == 0:
dist_membr_RNA = np.copy(dist_nuc_RNA_loop)
dist_membr_pix = np.copy(dist_membr_pix_loop)
else:
dist_membr_RNA = np.append(dist_membr_RNA,
dist_nuc_RNA_loop,
axis=0)
dist_membr_pix = np.append(dist_membr_pix,
dist_membr_pix_loop,
axis=0)
idx += 1
# *************************************************************************
# Load and analyze FQ results
xTicks = binsHist[:-1]
width = 0.9 * np.diff(binsHist)
width_full = np.diff(binsHist)
center = (binsHist[:-1] + binsHist[1:]) / 2
histRNA_all, bins = np.histogram(dist_membr_RNA, binsHist, density=False)
histPIX_all, bins = np.histogram(dist_membr_pix, binsHist, density=False)
histRNA_norm = np.divide(histRNA_all, histPIX_all)
counts_total = np.nansum(np.multiply(histRNA_norm, width_full))
histRNA_norm = np.divide(histRNA_norm, counts_total)
fig1, ax = plt.subplots(3, 1)
ax[0].bar(center, histRNA_all, align='center', width=width)
ax[0].set_xlabel('Dist to nuc')
ax[0].set_ylabel('# RNAs')
ax[0].set_xticks(xTicks)
ax[0].set_xticklabels(xTicks.astype(int))
ax[1].bar(center, histPIX_all, align='center', width=width)
ax[1].set_xlabel('Dist to nuc')
ax[1].set_ylabel('# pixels')
ax[1].set_xticks(xTicks)
ax[1].set_xticklabels(xTicks.astype(int))
ax[2].bar(center, histRNA_norm, align='center', width=width)
ax[2].set_xlabel('Distance to nuc')
ax[2].set_ylabel('RNA counts [a.u.]')
ax[2].set_xticks(xTicks)
ax[2].set_xticklabels(xTicks.astype(int))
ax[0].title.set_text('RNAs')
ax[1].title.set_text('All pixel')
ax[2].title.set_text('RNA renormalized with pixels')
plt.tight_layout()
#%% Read nuclear outlines
importlib.reload(annotationImporter)
importlib.reload(maskGenerator)
## Open FQ results
file_load = '/Volumes/PILON_HD2/fmueller/Documents/Data/ImJoy/rna-loc/NucMembEnrichment/C1-N2_imb-2-670_NG-610_04_R3D_res_GMM.txt'
## Get folders
drive, path_and_file = os.path.splitdrive(file_load)
path_results, file_results = os.path.split(path_and_file)
## Import annotations
path_annot = os.path.join(drive, path_results, 'zstack_segmentation')
folderImporter = annotationImporter.FolderImporter(channels={'nuclei': 'C4-'},
data_category={'roi': ''},
annot_ext='__RoiSet.zip',
progress_callback=None)
annotDict = folderImporter.load(path_annot)
print('average roi size:', annotDict['roi_size'])
# Generate binary masks for a selected data-set
binaryGen = maskGenerator.BinaryMaskGenerator(erose_size=5,
obj_size_rem=500,
save_indiv=True,
progress_callback=None)
# The generate function uses as an input the sub-dictionary for one data-category and one channel
annotatFiles = annotDict['roi']['nuclei']
maskDict = binaryGen.generate(annotatFiles)
# Use a loop and the update function to add the mask dictionary to the loaded annotation dictionary
def process_folder_fiji(path_open, channels, img_ext, annot_ext, masks_save):
'''
Function uses annotations generated in FIJI and creates mask based
on the specified parameters. The resulting files are zipped and be
used for training of a neural network with ImJoy
'''
## Create folder to save results
path_save_unet = os.path.join(path_open, 'unet_data_tmp')
create_folder(path_save_unet)
# Load data with FolderImporter
folderImporter = annotationImporter.FolderImporter(channels=channels,
data_category={
'train': 'train',
'test': 'test'
},
img_ext=img_ext,
annot_ext=annot_ext)
annotDict = folderImporter.load(path_open)
print('average roi size:', annotDict['roi_size'])
# Generate binary masks
binaryGen = maskGenerator.BinaryMaskGenerator(erose_size=5,
obj_size_rem=500,
save_indiv=False)
# Loop over data categories and verify if corresponding folder exists
for key_categ, categ in annotDict['config']['data_category'].items():
# Folder to save category
path_save_categ = os.path.join(path_save_unet, key_categ)
create_folder(path_save_categ)
# Loop over channels in data-categ
for key_channel, channel in annotDict['config']['channels'].items():
# The generate function uses as an input the sub-dictionary for one data-category and one channel
maskDict = binaryGen.generate(annotDict[key_categ][key_channel])
# Loop over masks and save specified ones
for key_file, v in maskDict.items():
for key_mask_sel in masks_save[key_channel]:
## Get file name without extension
file_base, ext = os.path.splitext(key_file)
## Save label
img_save = maskDict[key_file][key_mask_sel]
file_name_save = os.path.join(
path_save_categ,
'{}_{}.png'.format(file_base, key_mask_sel))
imsave(file_name_save, img_save)
## Save image
img_save = annotDict[key_categ][key_channel][key_file][
'image']
file_name_save = os.path.join(path_save_categ, key_file)
imsave(file_name_save, img_save)
# Create zip file and delete original folder
file_name_zip = os.path.join(path_open, 'unet_data')
shutil.make_archive(file_name_zip, 'zip', path_save_unet)
if os.path.isdir(path_save_unet):
shutil.rmtree(path_save_unet)
def process_file(FQ_file,
bin_prop=(0, 90, 20),
channels={'cells': 'C3-'},
data_category={'roi': ''},
annotation_extension='__RoiSet.zip',
img_extension='.tif',
show_plots=False,
Zrange=None,
dZ=2,
plot_callback=None,
progress_callback=None,
log_callback=None):
'''
Enrichment along the CELL MEMBRANE
Function uses annotations generated in FIJI and creates mask based
on the specified parameters.
Args:
plot_callback ... callback function to plot results (e.g. in ImJoy interface)
Zrange [tuple, 2 elements]. [Optional] Tuple specifying minimum and maximum z-value that is considered
in analysis.
bin_prop [Tuple, 3 elements]. Specifies the bins for the histograms (min, max,delta).
'''
# Get input args. Has to be FIRST call!
input_args = locals()
input_args['plot_callback'] = str(input_args['plot_callback'])
input_args['progress_callback'] = str(input_args['progress_callback'])
input_args['log_callback'] = str(input_args['log_callback'])
# Show function name and input arguments
function_name = sys._getframe().f_code.co_name
utils.log_message(
f"Function ({function_name}) called with:\n {str(input_args)} ",
callback_fun=log_callback)
# Make sure input args are correct - assignments with 0 can come from ImJoy
if Zrange[0] == 0 or Zrange[1] == 0:
Zrange = None
if bin_prop[1] == 0 or bin_prop[1] == 0:
bin_prop = (0, 90, 20)
## Prepare folder to save results
drive, path_and_file = os.path.splitdrive(FQ_file)
path_results, file_results = os.path.split(path_and_file)
file_base, ext = os.path.splitext(file_results)
path_save = os.path.join(
drive, path_results, file_base,
'MembDist_{}'.format(time.strftime("%y%m%d-%H%M", time.localtime())))
utils.log_message(f"Save results in folder: {path_save}",
callback_fun=log_callback)
if not os.path.isdir(path_save):
os.makedirs(path_save)
## Open FQ results
fq_dict = FQtoolbox.read_FQ_matlab(FQ_file)
spots_all = FQtoolbox.get_rna(fq_dict)
Zrna = spots_all[:, [18]]
# Open annotations
utils.log_message(f'\n== Open annotations', callback_fun=log_callback)
if 'RoiSet.zip' in annotation_extension:
path_annot = os.path.join(drive, path_results, 'zstack_segmentation')
utils.log_message(f"Opening annotation folder: {path_annot}",
callback_fun=log_callback)
folderImporter = annotationImporter.FolderImporter(
channels=channels,
data_category=data_category,
annot_ext=annotation_extension,
progress_callback=progress_callback)
annotDict = folderImporter.load(path_annot)
str_size = str(annotDict['roi_size'])
utils.log_message(f'Average roi size: {str_size}',
callback_fun=log_callback)
#utils.log_message(f'{annotDict['roi_size']}', callback_fun=log_callback)
# Generate binary masks for a selected data-set
binaryGen = maskGenerator.BinaryMaskGenerator(
erose_size=5,
obj_size_rem=500,
save_indiv=True,
progress_callback=progress_callback)
# The generate function uses as an input the sub-dictionary for one data-category and one channel
annotatFiles = annotDict['roi']['cells']
maskDict = binaryGen.generate(annotatFiles)
# Use a loop and the update function to add the mask dictionary to the loaded annotation dictionary
#utils.log_message(str(maskDict.keys()), callback_fun=log_callback)
utils.log_message(f"Keys of mask dictionary: {str(maskDict.keys())} ",
callback_fun=log_callback)
keys_delete = []
for k, v in annotatFiles.items():
# Make sure that key in present in mask, otherwise delete
if k in maskDict:
v.update(maskDict[k])
else:
keys_delete.append(k)
utils.log_message(
f"Deleted keys of mask dictionary: {str(keys_delete)} ",
callback_fun=log_callback)
# Bins of histogram
binsHist = np.arange(bin_prop[0], bin_prop[1], bin_prop[2])
width = 0.8 * (binsHist[1] - binsHist[0])
center = (binsHist[:-1] + binsHist[1:]) / 2
# Other parameters for calculation
dist_membr_RNA = np.array([])
dist_membr_pix = np.array([])
idx = 0
# Loop over all z-slices
utils.log_message(f'\n== Loop over slices', callback_fun=log_callback)
hist_slice = {}
N_annot = len(annotatFiles)
for idx_file, (k_annot, v_annot) in enumerate(annotatFiles.items()):
# Indicate progress via callback
if progress_callback:
perc = int(100 * (idx_file + 1) / (N_annot))
progress_callback({
"task": "analyze_slices",
"text": f"{perc}%, {k_annot}",
"progress": perc
})
else:
#print(f'Slice {idx_file+1}/{N_annot}: {k_annot}')
utils.log_message(f'Slice {idx_file+1}/{N_annot}: {k_annot}',
callback_fun=log_callback)
# Get Z coordinate
m = re.search('.*_Z([0-9]*)\.tif', k_annot)
Zmask = int(m.group(1))
# Check if outside of specified z range
if Zrange is not None:
if (Zmask <= Zrange[0]) or (Zmask >= Zrange[1]):
print('Slice outside of range')
continue
# Get z-range for loop
if dZ > 0:
Zloop = np.logical_and(Zrna <= Zmask + dZ,
Zrna >= Zmask - dZ).flatten()
else:
Zloop = (Zrna == Zmask).flatten()
spots_loop = spots_all[Zloop, :]
spots_loop_XY = spots_loop[:, [16, 17]].astype(int)
# Distance transform
dist_membr = ndimage.distance_transform_edt(
~v_annot['mask_edge']) # Negate mask
# Indices have to be inversed to access array
dist_membr_RNA_loop = dist_membr[spots_loop_XY[:, 0], spots_loop_XY[:,
1]]
# Get distance from membrane for all pixel in the cell
mask_all = v_annot['mask_fill'] + v_annot['mask_edge']
dist_membr_pix_loop = dist_membr[mask_all]
# Save values
if idx == 0:
dist_membr_RNA = np.copy(dist_membr_RNA_loop)
dist_membr_pix = np.copy(dist_membr_pix_loop)
else:
dist_membr_RNA = np.append(dist_membr_RNA,
dist_membr_RNA_loop,
axis=0)
dist_membr_pix = np.append(dist_membr_pix,
dist_membr_pix_loop,
axis=0)
idx += 1
# Calculate histograms
histRNA, bins = np.histogram(dist_membr_RNA_loop,
binsHist,
density=False)
histpix, bins = np.histogram(dist_membr_pix_loop,
binsHist,
density=False)
histRNAnorm = histRNA / histRNA.sum()
histpixnorm = histpix / histpix.sum()
histRNAnormPix = np.divide(histRNAnorm, histpixnorm)
histRNAnormPix = np.nan_to_num(histRNAnormPix)
hist_plot = {
'width': width,
'center': center,
'bins': bins,
'histRNA': histRNA,
'histpix': histpix,
'histRNAnormPix': histRNAnormPix
}
hist_slice[f'Z{Zmask}'] = hist_plot
# Plot results
name_save = os.path.join(path_save, f'Z-{Zmask}.png')
plot_results_slice(Zmask, v_annot, mask_all, spots_loop_XY, dist_membr,
hist_plot, name_save, show_plots)
# Analyze all slices
histRNA_all, bins = np.histogram(dist_membr_RNA, binsHist, density=False)
histRNA_all_norm = histRNA_all / histRNA_all.sum()
# Renormalize with pixel counts
histpix_all, bins = np.histogram(dist_membr_pix, binsHist, density=False)
histpix_all_norm = histpix_all / histpix_all.sum()
hist_RNA_all_normPix = np.divide(histRNA_all_norm, histpix_all_norm)
hist_RNA_all_normPix = np.nan_to_num(hist_RNA_all_normPix)
hist_plot_all = {
'width': width,
'center': center,
'bins': bins,
'histRNA_all': histRNA_all,
'histRNA_all_norm': histRNA_all_norm,
'histpix_all_norm': histpix_all_norm,
'hist_RNA_all_normPix': hist_RNA_all_normPix
}
name_save = os.path.join(path_save, '_DistanceEnrichmentSummary.png')
plot_results_all(hist_plot_all, name_save, show_plots)
if plot_callback:
plot_callback(name_save)
if progress_callback:
with open(name_save, 'rb') as f:
data = f.read()
result = base64.b64encode(data).decode('ascii')
imgurl = 'data:image/png;base64,' + result
progress_callback({"task": "show_results", "src": imgurl})
# Save entire analysis results as json (remove callback functions, since those cause errors)
input_args.pop('show_plot', None)
input_args.pop('plot_callback', None)
input_args.pop('progress_callback', None)
input_args.pop('log_callback', None)
analysis_results = {
'args': input_args,
'hist_all': hist_plot_all,
'hist_slice': hist_slice
}
name_json = os.path.join(path_save, 'DataAll.json')
with open(name_json, 'w') as fp:
json.dump(analysis_results,
fp,
sort_keys=True,
indent=4,
cls=utils.NumpyEncoder)
# Save histogram of pooled data as csv
name_csv = os.path.join(path_save, '_HistogramPooled.csv')
hist_plot_all.pop('bins', None)
hist_plot_all.pop('width', None)
csv_header = ';'.join(hist_plot_all.keys())
hist_values = np.array(list(hist_plot_all.values())).transpose()
np.savetxt(name_csv,
hist_values,
delimiter=";",
fmt='%f',
header=csv_header,
comments='')
return analysis_results
def calc_nuclear_enrichment(FQ_file,
binsHist,
channels={'nuclei': ''},
show_plots=False,
Zrange=None,
dZ=2,
plot_callback=None,
progress_callback=None,
log_callback=None):
"""
Enrichment at the nuclear MEMBRANE
Function uses annotations generated in FIJI and creates mask based
on the specified parameters.
"""
# Get input args. Has to be FIRST call!
input_args = locals()
# Get folder
drive, path_and_file = os.path.splitdrive(FQ_file)
path_results, file_results = os.path.split(path_and_file)
file_base, ext = os.path.splitext(file_results)
path_save = os.path.join(
drive, path_results, file_base, 'NucEnvelopDist_{}'.format(
time.strftime("%y%m%d-%H%M", time.localtime())))
if not os.path.isdir(path_save):
os.makedirs(path_save)
# *************************************************************************
# Load nuclear outline
utils.log_message(f'Reading segmentation masks of nuclei',
callback_fun=log_callback)
# Generate binary masks for a selected data-set
binaryGen = maskGenerator.BinaryMaskGenerator(
erose_size=5,
obj_size_rem=500,
save_indiv=True,
progress_callback=progress_callback)
# Import annotations for nuclei
path_annot = os.path.join(drive, path_results, 'zstack_segmentation')
folderImporter = annotationImporter.FolderImporter(
channels=channels,
data_category={'roi': ''},
annot_ext='__RoiSet.zip',
progress_callback=progress_callback)
annotDict = folderImporter.load(path_annot)
# The generate function uses as an input the sub-dictionary for one data-category and one channel
annotatFiles = annotDict['roi']['nuclei']
mask_dict_nuclei = binaryGen.generate(annotatFiles)
keys_delete = []
for k, v in annotatFiles.items():
# Make sure that key in present in mask, otherwise delete
if k in mask_dict_nuclei:
v.update(mask_dict_nuclei[k])
else:
keys_delete.append(k)
# *************************************************************************
# Load embryo outline
file_embryo = os.path.join(drive, path_results, 'embryo_contour.roi')
# Convert dictionary & get size of ROIS
roi_import = read_roi_file(file_embryo)
roi_dict = {}
roi_dict['embryo'] = {}
roi_dict['embryo']['type'] = roi_import['embryo_contour']['type']
roi_dict['embryo']['pos'] = np.column_stack(
(roi_import['embryo_contour']['y'], roi_import['embryo_contour']['x']))
# Assemble structure to call mask generator
image_size = annotatFiles.values().__iter__().__next__()['image'].shape
image_fake = np.zeros(image_size, dtype=np.uint8)
annotat_files_embryo = {}
annotat_files_embryo['embryo_contour'] = {}
annotat_files_embryo['embryo_contour']['roi'] = roi_dict
annotat_files_embryo['embryo_contour']['image'] = image_fake
mask_dict_embryo = binaryGen.generate(annotat_files_embryo)
mask_embryo = mask_dict_embryo['embryo_contour']['mask_fill']
# *************************************************************************
# Load and analyze FQ results
fq_dict = FQtoolbox.read_FQ_matlab(FQ_file)
spots_all = FQtoolbox.get_rna(fq_dict)
# Z position in pixel
Zrna = np.divide(spots_all[:, [2]],
fq_dict['settings']['microscope']['pix_z']).astype(int)
# Other parameters for calculation
dist_membr_RNA = np.array([])
dist_membr_pix = np.array([])
idx = 0
# Loop over all z-slices
utils.log_message(f'Loop over z-slices', callback_fun=log_callback)
N_annot = len(annotatFiles)
for idx_file, (k_annot, v_annot) in enumerate(annotatFiles.items()):
# Indicate progress via callback
if progress_callback:
perc = int(100 * (idx_file + 1) / (N_annot))
progress_callback({
"task": "analyze_slices",
"text": f"{perc}%, {k_annot}",
"progress": perc
})
else:
print(f'Slice: {k_annot}')
# Get Z coordinate
m = re.search('.*_Z([0-9]*)\.tif', k_annot)
Zmask = int(m.group(1))
# Check if outside of specified z range
if Zrange is not None:
if not (Zrange[0] == 0 and Zrange[1] == 0):
if (Zmask < Zrange[0]) or (Zmask > Zrange[1]):
print(f'Z-slice outside of specified range: {Zmask}')
continue
# Get z-range for loop
if dZ == 0:
Zloop = np.logical_and(Zrna <= Zmask + dZ,
Zrna >= Zmask - dZ).flatten()
else:
Zloop = (Zrna == Zmask).flatten()
spots_loop = spots_all[Zloop, :]
spots_loop_XY = np.divide(
spots_loop[:, [0, 1]],
fq_dict['settings']['microscope']['pix_xy']).astype(int)
# Distance transform
dist_nuc_outside = ndimage.distance_transform_edt(
~v_annot['mask_fill'])
dist_nuc_inside = ndimage.distance_transform_edt(
v_annot['mask_fill']) # Negate mask
dist_nuc = dist_nuc_outside - dist_nuc_inside
# Indices have to be inverted to access array
dist_nuc_RNA_loop = dist_nuc[spots_loop_XY[:, 0], spots_loop_XY[:, 1]]
# Get distance from membrane for all pixel in the cell
dist_membr_pix_loop = dist_nuc[mask_embryo.astype(bool)]
# Save values
if idx == 0:
dist_membr_RNA = np.copy(dist_nuc_RNA_loop)
dist_membr_pix = np.copy(dist_membr_pix_loop)
else:
dist_membr_RNA = np.append(dist_membr_RNA,
dist_nuc_RNA_loop,
axis=0)
dist_membr_pix = np.append(dist_membr_pix,
dist_membr_pix_loop,
axis=0)
idx += 1
# *************************************************************************
# Load and analyze FQ results
utils.log_message(f'Analyzing smFISH data', callback_fun=log_callback)
x_ticks = binsHist[:-1]
width = 0.9 * np.diff(binsHist)
width_full = np.diff(binsHist)
center = (binsHist[:-1] + binsHist[1:]) / 2
#histRNA_all, bins = np.histogram(dist_membr_RNA,binsHist ,density=False)
#histPIX_all, bins = np.histogram(dist_membr_pix,binsHist ,density=False)
#histRNA_norm = np.divide(histRNA_all,histPIX_all)
#counts_total = np.nansum(np.multiply(histRNA_norm,width_full))
#histRNA_norm = np.divide(histRNA_norm,counts_total)
# Distance of all pixel from distance map
hist_pix_all, bins = np.histogram(dist_membr_pix, binsHist, density=False)
hist_pix_all_norm = hist_pix_all / hist_pix_all.sum()
# Distance of all RNAs
hist_rna_all, bins = np.histogram(dist_membr_RNA, binsHist, density=False)
hist_rna_all_norm = hist_rna_all / hist_rna_all.sum()
# Re-normalize RNA distances
hist_rna_all_norm_pix = np.divide(hist_rna_all_norm, hist_pix_all_norm)
hist_rna_all_norm_pix = np.nan_to_num(hist_rna_all_norm_pix)
# Save file with histogram
histo_dist = {
'center': center,
'width': width_full,
'bins': binsHist,
'x_ticks': x_ticks,
'hist_pix_all': hist_pix_all,
'hist_pix_all_norm': hist_pix_all_norm,
'hist_rna_all': hist_rna_all,
'hist_rna_all_norm': hist_rna_all_norm,
'hist_rna_all_norm_pix': hist_rna_all_norm_pix
}
# Save entire analysis results as json
input_args.pop('show_plot', None)
input_args.pop('plot_callback', None)
input_args.pop('progress_callback', None)
input_args.pop('log_callback', None)
analysis_results = {
'args': input_args,
'histogram': histo_dist,
}
name_json = os.path.join(path_save, 'DataAnalysis.json')
with open(name_json, 'w') as fp:
json.dump(analysis_results,
fp,
sort_keys=True,
indent=4,
cls=utils.NumpyEncoder)
# Save histogram of pooled data as csv
name_csv = os.path.join(path_save, '_HistogramDistances.csv')
histo_dist.pop('bins', None)
csv_header = ';'.join(histo_dist.keys())
hist_values = np.array(list(histo_dist.values())).transpose()
np.savetxt(name_csv,
hist_values,
delimiter=";",
fmt='%f',
header=csv_header,
comments='')
# *************************************************************************
# Plot results
# Don't show plots when they are saved
if not show_plots:
plt.ioff()
fig1, ax = plt.subplots(3, 1)
ax[0].bar(center, hist_rna_all, align='center', width=width)
ax[0].set_xlabel('Dist to nuc')
ax[0].set_ylabel('# RNAs')
ax[0].set_xticks(x_ticks)
ax[0].set_xticklabels(x_ticks.astype(int))
ax[1].bar(center, hist_pix_all, align='center', width=width)
ax[1].set_xlabel('Dist to nuc')
ax[1].set_ylabel('# pixels')
ax[1].set_xticks(x_ticks)
ax[1].set_xticklabels(x_ticks.astype(int))
ax[2].bar(center, hist_rna_all_norm_pix, align='center', width=width)
ax[2].set_xlabel('Distance to nuc')
ax[2].set_ylabel('RNA counts [a.u.]')
ax[2].set_xticks(x_ticks)
ax[2].set_xticklabels(x_ticks.astype(int))
ax[0].title.set_text('RNAs')
ax[1].title.set_text('All pixel in images')
ax[2].title.set_text('RNAs renormalized with pixels')
plt.tight_layout()
# Save and close if display not required
name_save = os.path.join(path_save, '_DistanceEnrichmentSummary.png')
plt.savefig(name_save, dpi=300)
if not show_plots:
plt.close()
if plot_callback:
plot_callback(name_save)
if progress_callback:
with open(name_save, 'rb') as f:
data = f.read()
result = base64.b64encode(data).decode('ascii')
img_url = 'data:image/png;base64,' + result
progress_callback({"task": "show_results", "src": img_url})