def plot_traces_multiple_evaluated(row = None, session_wise = False):
'''
Plots different versions of contour images that change the inicialization parameters for source extraccion.
The idea is to see the impact of different seed selection in the final source extraction result.
:param row: one analysis state row
:return: figure
'''
corr_min = round(eval(row['source_extraction_parameters'])['min_corr'],1)
pnr_min = round(eval(row['source_extraction_parameters'])['min_pnr'],1)
r_min = eval(row['component_evaluation_parameters'])['rval_thr']
snf_min = eval(row['component_evaluation_parameters'])['min_SNR']
output_source_extraction = eval(row.loc['source_extraction_output'])
corr_path = output_source_extraction['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
cnm_file_path = output_source_extraction['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
C = cnm.estimates.C
output_component_evaluation = eval(row.loc['component_evaluation_output'])
cnm_file_path = output_component_evaluation['main']
cnm_eval = load_CNMF(db.get_file(cnm_file_path))
idx = cnm_eval.estimates.idx_components
idx_b = cnm_eval.estimates.idx_components_bad
fig, ax = plt.subplots(1)
C[0] += C[0].min()
for i in range(1, len(C)):
C[i] += C[i].min() + C[:i].max()
if i in idx_b:
color = 'red'
else:
color = 'blue'
ax.plot(C[i],color = color)
ax.set_xlabel('t [frames]')
ax.set_yticks([])
ax.set_ylabel('activity')
ax.set_title('Corr=' + f'{corr_min}' + ', PNR = ' + f'{pnr_min}' + ', PCC = ' + f'{r_min}' + ', SNR =' + f'{snf_min}')
fig.set_size_inches([10., .3 * len(C)])
fig_dir = 'data/interim/component_evaluation/trial_wise/meta/figures/traces/'
if session_wise:
fig_dir = 'data/interim/component_evaluation/session_wise/meta/figures/traces/'
fig_name = fig_dir + db.create_file_name(5,row.name) + '.png'
fig.savefig(fig_name)
return
def plot_contours_evaluated(row = None, session_wise = False):
'''
Plot contours for all cells, selected cells and non selected cells, and saves it in
figure_path = '/data/interim/component_evaluation/trial_wise/meta/figures/contours/'
:param row: one analysis state row
'''
index = row.name
corr_min = round(eval(row['source_extraction_parameters'])['min_corr'],1)
pnr_min = round(eval(row['source_extraction_parameters'])['min_pnr'],1)
r_min = eval(row['component_evaluation_parameters'])['rval_thr']
snf_min = eval(row['component_evaluation_parameters'])['min_SNR']
output_source_extraction = eval(row.loc['source_extraction_output'])
corr_path = output_source_extraction['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
output_component_evaluation = eval(row.loc['component_evaluation_output'])
cnm_file_path = output_component_evaluation['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
figure, axes = plt.subplots(1, 3)
axes[0].imshow(cn_filter)
axes[1].imshow(cn_filter)
axes[2].imshow(cn_filter)
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[0].plot(*v.T, c='w')
axes[0].set_title('All components')
axes[0].set_ylabel('Corr=' + f'{corr_min}' + ', PNR = ' + f'{pnr_min}' + ', PCC = ' + f'{r_min}' + ', SNR =' + f'{snf_min}')
idx = cnm.estimates.idx_components
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx], np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[1].plot(*v.T, c='b')
axes[1].set_title('Accepted components')
idx_b = cnm.estimates.idx_components_bad
coordinates_b = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx_b], np.shape(cn_filter), 0.2, 'max')
for c in coordinates_b:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[2].plot(*v.T, c='r')
axes[2].set_title('Rejected components')
figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/trial_wise/meta/figures/contours/'
if session_wise:
figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/session_wise/meta/figures/contours/'
figure_name = figure_path + db.create_file_name(5,index) + '.png'
figure.savefig(figure_name)
return figure
def get_fig_session_wise_neurons_per_trial_histogram(mouse, session):
# Get selected rows
selected_rows = src.pipeline.select('component_evaluation', mouse, session)
nr_components_list = []
# Get the number of components for each trial
for index, row in selected_rows.iterrows():
try:
cnm = load_CNMF(
src.pipeline.get_file(
eval(row.loc['source_extraction_output'])['main']))
nr_components_list.append(len(cnm.estimates.C))
except:
pass
# Number of trials
N = len(nr_components_list)
fig, ax = plt.subplots(1)
plt.hist(nr_components_list)
plt.ylabel('# trials')
plt.xlabel('K')
print('N = ', N)
return fig
def get_fig_session_wise_neurons_per_trial(mouse, session):
# Get selected rows
selected_rows = src.pipeline.select('component_evaluation', mouse, session)
trial_string_list = []
nr_components_list = []
# Get the number of components for each trial
for index, row in selected_rows.iterrows():
mouse, session, trial, is_rest, *r = index
trial_string = str(trial) + '_R' if bool(is_rest) else str(trial)
trial_string_list.append(trial_string)
if type(row.loc['source_extraction_output']) is str:
cnm_file_path = eval(row.loc['source_extraction_output'])['main']
else:
cnm_file_path = f'data/interim/source_extraction/trial_wise/main/{src.pipeline.create_file_name(step_index, index)}.hdf5'
cnm = load_CNMF(cnm_file_path)
nr_components_list.append(len(cnm.estimates.C))
# Number of trials
N = len(nr_components_list)
fig, ax = plt.subplots(1)
# fig.size(N * 50, 500)
x_pos = np.arange(N)
plt.bar(x_pos, nr_components_list)
plt.ylabel('# components')
plt.xticks(x_pos, trial_string_list)
print('N = ', N)
return fig
def load_results(results_dir):
# Determine filename from dir
fn = os.path.join(results_dir, 'analysis_results.hdf5')
# Load CNMF object using CaImAn function
cnm = cnmf.load_CNMF(fn)
print('Number of components: {}'.format(cnm.estimates.C.shape[0]))
return cnm
def plot_traces_multiple(row,
version=None,
corr_array=None,
pnr_array=None,
session_wise=False):
'''
Plots different versions of contour images that change the inicialization parameters for source extraccion.
The idea is to see the impact of different seed selection in the final source extraction result.
:param row: one analysis state row
:param version: array containing the version numbers of source extraction that will be ploted
:param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions
:param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions
:param: session_wise bool that indicates where the figure is save
:return: None
'''
states_df = db.open_analysis_states_database()
index = row.name
for ii in range(corr_array.shape[0]):
for jj in range(pnr_array.shape[0]):
fig, ax = plt.subplots(1)
new_row = db.select(
states_df,
'component_evaluation',
mouse=index[0],
session=index[1],
trial=index[2],
is_rest=index[3],
cropping_v=index[5],
motion_correction_v=index[6],
alignment_v=index[7],
source_extraction_v=version[ii * len(pnr_array) + jj])
new_row = new_row.iloc[0]
output = eval(new_row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
C = cnm.estimates.C
idx_components = cnm.estimates.idx_components
C[0] += C[0].min()
for i in range(1, len(C)):
C[i] += C[i].min() + C[:i].max()
ax.plot(C[i])
ax.set_xlabel('t [frames]')
ax.set_yticks([])
ax.set_ylabel('activity')
fig.set_size_inches([10., .3 * len(C)])
fig_dir = 'data/interim/source_extraction/trial_wise/meta/figures/traces/'
if session_wise:
fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/traces/'
fig_name = fig_dir + db.create_file_name(
3, new_row.name
) + '_corr_min' + f'{round(corr_array[ii], 1)}' + '_pnr_min' + f'{round(pnr_array[jj], 1)}' + '_.png'
fig.savefig(fig_name)
return
def plot_session_contours(selected_rows, version=None, corr_array=None, pnr_array=None):
'''
Plots different versions of contour images that change the initialization parameters for source extraction.
The idea is to see the impact of different seed selection in the final source extraction result.
:param selected_rows: rows corresponding to different trials
:param version: array containing the version numbers of source extraction that will be plotted
:param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions
:param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions
:return: (saves multiple figures)
'''
states_df = db.open_analysis_states_database()
for ii in range(corr_array.shape[0]):
for jj in range(pnr_array.shape[0]):
figure, axes = plt.subplots(1, len(selected_rows), figsize=(50, 10))
for i in range(len(selected_rows)):
row = selected_rows.iloc[i]
index = row.name
new_row = db.select(states_df, 'component_evaluation', mouse=index[0], session=index[1],
trial=index[2], is_rest=index[3], cropping_v=index[5], motion_correction_v=index[6],
alignment_v=index[7], source_extraction_v=version[ii * len(pnr_array) + jj])
new_row = new_row.iloc[0]
output = eval(new_row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
corr_path = output['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
# axes[i].imshow(np.clip(cn_filter, min_corr, max_corr), cmap='viridis')
axes[i].imshow(cn_filter)
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[i].plot(*v.T, c='w')
axes[i].set_title('Trial = ' + f'{i + 1}', fontsize=30)
axes[i].set_xlabel('#cells = ' + f'{cnm.estimates.A.shape[1]}', fontsize=30)
figure.suptitle('min_corr = ' + f'{round(corr_array[ii], 2)}' + 'min_pnr = ' + f'{round(pnr_array[jj], 2)}',
fontsize=50)
fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/'
fig_name = fig_dir + db.create_file_name(3,
new_row.name) + '_version_' + f'{version[ii * len(pnr_array) + jj]}' + '.png'
figure.savefig(fig_name)
return
def plot_source_extraction_result(mouse_row_new):
'''
Generates and saves a contour plot and a trace plot for the specific mouse_row
'''
corr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_corr'], 1)
pnr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_pnr'], 1)
output_source_extraction = eval(mouse_row_new.loc['source_extraction_output'])
corr_path = output_source_extraction['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
cnm_file_path = output_source_extraction['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
figure, axes = plt.subplots(1)
axes.imshow(cn_filter)
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes.plot(*v.T, c='w')
axes.set_title('min_corr = ' + f'{corr_min}')
axes.set_ylabel('min_pnr = ' + f'{pnr_min}')
fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/'
file_name = db.create_file_name(3, mouse_row_new.name)
figure.savefig(fig_dir + file_name + '.png')
## up to here
fig, ax = plt.subplots(1)
C = cnm.estimates.C
C[0] += C[0].min()
for i in range(1, len(C)):
C[i] += C[i].min() + C[:i].max()
ax.plot(C[i])
ax.set_xlabel('t [frames]')
ax.set_yticks([])
ax.set_ylabel('activity')
fig.set_size_inches([10., .3 * len(C)])
fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/traces/'
fig_name = fig_dir + db.create_file_name(3, mouse_row_new.name) + '.png'
fig.savefig(fig_name)
return
def plot_source_extraction_result_specific_cell(mouse_row_new, cell_number):
'''
(Still need to be finished) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
THERE IS AN ERROR IN THE
In the first plot shows correlation image and contour of the selected neurons.
In the second plot shows the traces for the selected neurons.
:param mouse_row_new: data base row
:param cell_number: array with the cells that are selected to be ploted
:return: None
'''
corr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_corr'], 1)
pnr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_pnr'], 1)
output_source_extraction = eval(mouse_row_new.loc['source_extraction_output'])
corr_path = output_source_extraction['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
cnm_file_path = output_source_extraction['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
f, (a0, a1) = plt.subplots(2, 1, gridspec_kw={'height_ratios': [3, 1]})
a0.imshow(cn_filter)
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for i in cell_number:
v = coordinates[i]['coordinates']
coordinates[i]['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
a0.plot(*v.T, c='w')
a0.set_title('Contour Plot')
fig, ax = plt.subplots(1)
C = cnm.estimates.C
C[0] += C[0].min()
for i in range(cell_number):
C[i] += C[i].min() + C[:i].max()
a1.plot(C[i])
a1.set_xlabel('t [frames]')
a1.set_yticks([])
a1.set_title('Calcium Traces')
fig.set_size_inches([10., .3 * len(C)])
fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/'
fig_name = fig_dir + db.create_file_name(3, mouse_row_new.name) + '_example.png'
f.savefig(fig_name)
return
def plot_multiple_contours_session_wise(selected_rows, version = None , corr_array = None, pnr_array = None):
'''
Plots different versions of contour images that change the initialization parameters for source extraction.
The idea is to see the impact of different seed selection in the final source extraction result.
:param selected_rows: all analysis state selected
:param version: array containing the version numbers of source extraction that will be plotted
:param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions
:param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions
:return: figure
'''
states_df = db.open_analysis_states_database()
figure, axes = plt.subplots(len(corr_array), len(pnr_array), figsize=(15, 15))
color = ['w','b','r','m','c']
for row in range(len(selected_rows)):
mouse_row = selected_rows.iloc[row]
index = mouse_row.name
output = eval(mouse_row.loc['source_extraction_output'])
corr_path = output['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
for ii in range(corr_array.shape[0]):
for jj in range(pnr_array.shape[0]):
axes[ii, jj].imshow(cn_filter)
new_row = db.select(states_df, 'component_evaluation', mouse=index[0], session=index[1],
trial=index[2], is_rest=index[3], cropping_v=index[5], motion_correction_v=index[6],
source_extraction_v=version[ii * len(pnr_array) + jj])
new_row = new_row.iloc[0]
output = eval(new_row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[ii, jj].plot(*v.T, c = color[row])
axes[ii, jj].set_title('min_corr = ' + f'{round(corr_array[ii],2)}')
axes[ii, jj].set_ylabel('min_pnr = ' + f'{round(pnr_array[jj],2)}')
fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/'
fig_name = fig_dir + db.create_file_name(3, new_row.name)+'_corr_min' + f'{round(corr_array[0],1)}'+ '_pnr_min'+f'{round(pnr_array[0],1)}' + '_all.png'
figure.savefig(fig_name)
return figure
def plot_multiple_contours(rows, version = None , corr_array = None, pnr_array = None,session_wise = False):
'''
Plots different versions of contour images that change the initialization parameters for source extraction.
The idea is to see the impact of different seed selection in the final source extraction result.
:param row: one analysis state row
:param version: array containing the version numbers of source extraction that will be plotted
:param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions
:param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions
:return: figure
'''
figure, axes = plt.subplots(len(corr_array), len(pnr_array), figsize=(15, 15))
for ii in range(corr_array.shape[0]):
for jj in range(pnr_array.shape[0]):
version_number = ii *corr_array.shape[0] + jj + 1
if version_number in version:
new_row = rows.query('(source_extraction_v == ' + f'{version_number}' + ')')
new_row = new_row.iloc[0]
output = eval(new_row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
corr_path = output['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
axes[ii, jj].imshow(cn_filter)
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[ii, jj].plot(*v.T, c='w')
axes[ii, jj].set_title('min_corr = ' + f'{round(corr_array[ii],2)}')
axes[ii, jj].set_ylabel('min_pnr = ' + f'{round(pnr_array[jj],2)}')
fig_dir = 'data/interim/source_extraction/trial_wise/meta/figures/contours/'
if session_wise:
fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/'
fig_name = fig_dir + db.create_file_name(3, new_row.name)+'_corr_min' + f'{round(corr_array[0],1)}'+ '_pnr_min'+f'{round(pnr_array[0],1)}' + '_.png'
figure.savefig(fig_name)
return figure
def traces_extraction_AGONIA(data_path,
agonia_th,
neuropil_contaminated=False):
'''Extract traces of boxes using agonia Extractor class'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load Caiman results
cnm = cnmf.load_CNMF(results_caiman_path)
# calculate the centers of the CaImAn factors
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
# keep only cells above confidence threshold
boxes = boxes[boxes[:, 4] > agonia_th].astype('int')
k = 0
for cell, box in enumerate(boxes):
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
if not idx_factor:
boxes = np.delete(boxes, cell - k, axis=0)
k += 1
ola = Extractor()
for frame in images:
ola.extract(frame, boxes)
traces, traces_with_bg = ola.get_traces()
ola.pool.terminate()
if neuropil_contaminated:
return traces.T, traces_with_bg.T
else:
return traces.T
def boxes_exploration_interactive(data_path):
'''Returns an interactive plot with the agonia boxes with a confidence value above
a Score selected by the user with a slider. The user can select a box by clicking on it
and mean box Fluorescence and the Caiman DF/F of such box will be ploted.'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
cnm = cnmf.load_CNMF(results_caiman_path)
img = hv.Image(median_projection,
bounds=(0, 0, median_projection.shape[1],
median_projection.shape[0])).options(cmap='gray')
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
#scatter = hv.Scatter((centers[:,1], median_projection.shape[0] - centers[:,0]))
kdims = [hv.Dimension('Score', values=np.arange(0.05, 1, 0.05))]
tap = streams.SingleTap(transient=True, source=img)
Experiment = Stream.define('Experiment', data_path=data_path)
Centers = Stream.define('Centers', centers=centers)
CaImAn_detection = Stream.define('CaImAn_detection', cnm=cnm)
dmap = hv.DynamicMap(plot_AGonia_boxes_interactive,
kdims=kdims,
streams=[Experiment(), tap])
dmap1 = hv.DynamicMap(plot_boxes_traces,
kdims=kdims,
streams=[Experiment(), tap])
dmap2 = hv.DynamicMap(
plot_seeded_traces,
kdims=kdims,
streams=[CaImAn_detection(),
Experiment(), tap,
Centers()])
return ((img * dmap).opts(width=500, height=500) +
dmap1.opts(width=500, height=250) +
dmap2.opts(width=500, height=250)).opts(
opts.Curve(framewise=True)).cols(1)
def load_hdf5_result(result_dir,
gdrive_result_dir,
rclone_config,
use_filtered=False):
hdf5_filename = 'analysis_results.hdf5'
if use_filtered:
hdf5_filename = 'analysis_results_filtered.hdf5'
result_dir = os.path.join(result_dir, 'filtered')
gdrive_result_dir = os.path.join(gdrive_result_dir, 'filtered')
h5fpath = os.path.join(result_dir, hdf5_filename)
if not os.path.isfile(h5fpath):
success = gdrive_download_file(
os.path.join(gdrive_result_dir, hdf5_filename), result_dir,
rclone_config)
if not success:
return None
from caiman.source_extraction.cnmf.cnmf import load_CNMF
try:
return load_CNMF(h5fpath)
except OSError as e:
logging.error('Failed to open file: %s', h5fpath)
raise e
if not os.path.exists(outDir):
os.mkdir(outDir)
if not os.path.exists(outImageFolder):
os.mkdir(outImageFolder)
print('Input hdf5 file: {}'.format(hdf5File), flush=True)
print('Output directory: {}'.format(outDir), flush=True)
# load in data
print('Loading data', flush=True)
if args.online:
cnm = cnmf.online_cnmf.load_OnlineCNMF(hdf5File)
else:
cnm = cnmf.load_CNMF(hdf5File)
cnm.estimates.dims = cnm.dims
A, C, b, f = tp.cellInfoCaimanHdf5(hdf5File, dims=cnm.dims)
print('Calculating masks and stats. nSources: {}, maskThresh: {}'.format(
nSources, maskThresh),
flush=True)
meanImageSources = np.tensordot(C.mean(axis=0), A,
axes=(0, 0)).transpose().astype('float32')
meanImage = meanImageSources + np.tensordot(
f.mean(axis=0), b, axes=(0, 0)).transpose().astype('float32')
stdImageSources = np.tensordot(C.std(axis=0), A,
# %% imports
%matplotlib qt5
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import caiman as cm
from caiman.source_extraction.cnmf import cnmf as cnmf
from pathlib import Path
# %% load footprints
# path = "/media/georg/data/mesoscope/first data/with behavior/2021-08-24_day1_overview/reshaped/memmap__d1_1128_d2_1068_d3_1_order_C_frames_3901_sel.hdf5"
path = "/media/georg/data/mesoscope/first data/with behavior/2021-08-25_day2_square_1/reshaped/memmap__d1_1892_d2_2504_d3_1_order_C_frames_850_sel.hdf5"
cnmf2 = cnmf.load_CNMF(path)
# %% infer stuff
p = Path(cnmf2.mmap_file).stem.split('_')
xpx = int(p[p.index('d1')+1])
ypx = int(p[p.index('d2')+1])
nFrames = int(p[p.index('frames')+1])
# with open(folder.parent / 'meta.json','r') as fH:
# meta = json.load(fH)
# or set for now
fs = 3.03
# %%
cnmf2.estimates.detrend_df_f(quantileMin=8,frames_window=100)
D = cnmf2.estimates.F_dff
D_orig = copy(D)
D = D[good_inds,:]
'check_nan': check_nan,
'block_size_temp': block_size,
'block_size_spat': block_size,
'num_blocks_per_run_spat': num_blocks_per_run,
'num_blocks_per_run_temp': num_blocks_per_run,
'n_pixels_per_process': n_pixels_per_process,
'ssub': global_params['ssub'],
'tsub': global_params['tsub'],
'thr_method': 'nrg'
}
init_method = global_params['init_method']
opts = params.CNMFParams(params_dict=params_dict)
if reload:
cnm2 = load_CNMF(fname_new[:-5] + '_cnmf_perf_web.hdf5')
reproduce_images_website = False
if reproduce_images_website:
cnm2.estimates.evaluate_components(images,
params=cnm2.params,
dview=dview)
pl.figure(figsize=(10, 7))
good_snr = np.where(cnm2.estimates.SNR_comp > 5)[0]
examples = np.argsort(cnm2.estimates.cnn_preds)[-100:]
examples = np.intersect1d(examples, good_snr)[-5:]
for count, ex in enumerate(examples):
pl.subplot(6, 2, 2 * count + 1)
img = cnm2.estimates.A[:, ex].toarray().reshape(
cnm2.estimates.dims, order='F')
cm_ = np.array(
scipy.ndimage.measurements.center_of_mass(img)).astype(
def plot_multiple_contours_session_wise_evaluated(selected_rows):
## IN DEVELOPMENT!!!!!!!
'''
Plots different versions of contour images that change the initialization parameters for source extraction.
The idea is to see the impact of different seed selection in the final source extraction result.
:param selected_rows: all analysis state selected
:return: figure
'''
figure, axes = plt.subplots(3, 5, figsize=(50, 30))
for row in range(len(selected_rows)):
mouse_row = selected_rows.iloc[row]
index = mouse_row.name
output = eval(mouse_row.loc['source_extraction_output'])
corr_path = output['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
axes[0,row].imshow(cn_filter)
axes[1,row].imshow(cn_filter)
axes[2,row].imshow(cn_filter)
output = eval(mouse_row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[0,row].plot(*v.T, c = 'w', linewidth=3)
axes[0,row].set_title('Trial = ' + f'{row}')
axes[0,row].set_ylabel('')
output = eval(mouse_row.loc['component_evaluation_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
idx = cnm.estimates.idx_components
coordinates = cm.utils.visualization.get_contours(cnm.estimates.A[:, idx], np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[1,row].plot(*v.T, c='b', linewidth=3)
idx_b = cnm.estimates.idx_components_bad
coordinates_b = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx_b], np.shape(cn_filter), 0.2, 'max')
for c in coordinates_b:
v = c['coordinates']
c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))]
axes[2,row].plot(*v.T, c='r', linewidth=3)
source_extraction_parameters = eval(mouse_row['source_extraction_parameters'])
corr_lim = source_extraction_parameters['min_corr']
pnr_lim = source_extraction_parameters['min_pnr']
component_evaluation_parameters = eval(mouse_row['component_evaluation_parameters'])
pcc = component_evaluation_parameters['rval_thr']
SNR = component_evaluation_parameters['min_SNR']
figure.suptitle('Corr = ' + f'{corr_lim}' + 'PNR = ' + f'{pnr_lim}' + 'PCC = ' + f'{pcc}' + 'SNR = ' + f'{SNR}',
fontsize=50)
fig_dir = 'data/interim/component_evaluation/session_wise/meta/figures/contours/'
fig_name = fig_dir + db.create_file_name(3, index)+'_Corr = ' + f'{corr_lim}' + '_PNR = ' + f'{pnr_lim}' + '_PCC = ' + f'{pcc}' + '_SNR = ' + f'{SNR}' +'_.png'
figure.savefig(fig_name)
return figure
'cnn_lowest': 0.05
}
registration_params = {'max_thr': 0.45, 'thresh_cost': 0.75, 'max_dist': 10}
""" Prepare data """
gdrive_dated_dir = os.path.join(gdrive_subdir, exp_month, exp_title, exp_date)
gdrive_result_dir = os.path.join(gdrive_dated_dir, 'caiman', animal_name)
caiman_result_dir = os.path.join(local_miniscope_path, exp_title, exp_date,
'caiman', animal_name)
# Download result files if not stored locally
h5fpath = os.path.join(caiman_result_dir, 'analysis_results.hdf5')
if not os.path.isfile(h5fpath):
gdrive_download_file(gdrive_result_dir + '/analysis_results.hdf5',
caiman_result_dir, rclone_config)
cnm_obj = load_CNMF(h5fpath)
def filter_components(cnm_obj, components_quality_params, registration_params):
cnm_obj.estimates.threshold_spatial_components(
maxthr=registration_params['max_thr'])
cnm_obj.estimates.remove_small_large_neurons(
min_size_neuro=neuron_size_params['min'],
max_size_neuro=neuron_size_params['max'])
logging.info('filtered %d neurons based on the size',
len(cnm_obj.estimates.idx_components_bad))
cnm_obj.estimates.accepted_list = cnm_obj.estimates.idx_components
empty_images = np.zeros((1, ) + cnm_obj.dims)
if components_quality_params['use_cnn'] and (
not hasattr(cnm_obj.estimates, 'cnn_preds')
or len(cnm_obj.estimates.cnn_preds) == 0):
motion_correction_v=motion_correction_version,
alignment_v=0,
source_extraction_v=1)
row = selected_rows.iloc[1]
#input_mmap_file_path = eval(row.loc['alignment_output'])['main']
input_mmap_file_path = eval(row.loc['motion_correction_output'])['main']
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
# logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.')
images = Yr.T.reshape((T, ) + dims, order='F')
images = cm.movie(images)
output = eval(row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
W, b0 = cm.source_extraction.cnmf.initialization.compute_W(
Yr,
cnm.estimates.A.toarray(),
cnm.estimates.C,
cnm.estimates.dims,
1.4 * 5,
ssub=2)
cnm.estimates.W = W
cnm.estimates.b0 = b0
#movie_dir = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/processed/movies/'
#file_name = db.create_file_name(5,row.name)
#cnm.estimates.play_movie(cnm.estimates, images, movie_name= movie_dir + file_name + '.avi')
frame_range = slice(None, None, None)
Y_rec = cnm.estimates.A.dot(cnm.estimates.C[:, frame_range])
_x - x_,
fill=False,
color='r',
linestyle='--',
linewidth=3)
axes[kk, 0].add_patch(rect)
output_cropping = mouse_row['cropping_output']
cropped_file = eval(output_cropping)['main']
m = cm.load(cropped_file)
axes[kk, 1].imshow(m[0, :, :], cmap='gray')
output_source_extraction = eval(
mouse_row['source_extraction_output'])
cnm_file_path = output_source_extraction['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
corr_path = output_source_extraction['meta']['corr']['main']
cn_filter = np.load(db.get_file(corr_path))
axes[kk, 2].imshow(cn_filter)
coordinates = cm.utils.visualization.get_contours(
cnm.estimates.A, np.shape(cn_filter), 0.2, 'max')
for c in coordinates:
v = c['coordinates']
c['bbox'] = [
np.floor(np.nanmin(v[:, 1])),
np.ceil(np.nanmax(v[:, 1])),
np.floor(np.nanmin(v[:, 0])),
np.ceil(np.nanmax(v[:, 0]))
]
axes[kk, 2].plot(*v.T, c='w', linewidth=3)
def run_component_evaluation(input_file,
session_wise=False,
equalization=False):
sql = "SELECT source_extraction_session_wise,min_SNR,alignment_main,equalization_main,motion_correction_main,rval_thr,use_cnn FROM Analysis WHERE source_extraction_main=?"
val = [
input_file,
]
mycursor.execute(sql, val)
myresult = mycursor.fetchall()
data = []
aux = []
for x in myresult:
aux = x
for y in aux:
data.append(y)
if session_wise:
input_mmap_file_path = data[2]
elif equalization:
input_mmap_file_path = data[3]
else:
input_mmap_file_path = data[4]
parameters = {'min_SNR': data[1], 'rval_thr': data[5], 'use_cnn': data[6]}
data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/session_wise/' if \
data[0] else os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/trial_wise/'
sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,equalization_v,source_extraction_v,component_evaluation_v,input,home_path,decoding_main FROM Analysis WHERE source_extraction_main=?"
val = [
input_file,
]
mycursor.execute(sql, val)
result = mycursor.fetchall()
data = []
inter = []
for x in result:
inter = x
for y in inter:
data.append(y)
# Update the database
if data[10] == 0:
data[10] = 1
file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}"
output_file_path = f'main/{file_name}.hdf5'
sql1 = "UPDATE Analysis SET component_evaluation_main=?,component_evaluation_v=? WHERE source_extraction_main=? "
val1 = [output_file_path, data[10], input_file]
mycursor.execute(sql1, val1)
else:
data[10] += 1
file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}"
output_file_path = f'main/{file_name}.hdf5'
sql2 = "INSERT INTO Analysis (component_evaluation_main,component_evaluation_v) VALUES (?,?)"
val2 = [output_file_path, data[10]]
mycursor.execute(sql2, val2)
database.commit()
output_file_path_full = data_dir + output_file_path
# Load CNMF object
cnm = load_CNMF(input_mmap_file_path)
# Load the original movie
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
images = Yr.T.reshape((T, ) + dims, order='F')
# Set the parmeters
cnm.params.set('quality', parameters)
# Stop the cluster if one exists
n_processes = psutil.cpu_count()
try:
cm.cluster.stop_server()
except:
pass
# Start a new cluster
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local',
n_processes=n_processes,
# number of process to use, if you go out of memory try to reduce this one
single_thread=False)
# Evaluate components
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
logging.debug('Number of total components: ', len(cnm.estimates.C))
logging.debug('Number of accepted components: ',
len(cnm.estimates.idx_components))
# Stop the cluster
dview.terminate()
# Save CNMF object
cnm.save(output_file_path_full)
return output_file_path
if min_max is None:
min_ = img.min()
max_ = img.max()
else:
min_, max_ = min_max
img = (img-min_)/(max_-min_)*gain
img = img.astype(out_type)
img = np.dstack([img]*3)
return img
F = FileDialog()
# load object saved by CNMF
CNMF_filename = F.getOpenFileName(caption='Load CNMF Object',filter='*.*5')[0]
cnm_obj = load_CNMF(CNMF_filename)
# load movie
if not os.path.exists(cnm_obj.mmap_file):
M = FileDialog()
cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file', filter='*.mmap')[0]
mov = cm.load(cnm_obj.mmap_file)
min_mov = np.min(mov)
max_mov = np.max(mov)
# load summary image
Cn = cnm_obj.estimates.Cn
estimates = cnm_obj.estimates
def create_video(row, time_cropping, session_wise = False):
'''
This fuction creates a complete video with raw movie (motion corrected), source extracted cells and source extraction + background.
:param row: pandas dataframe containing the desired processing information to create the video. It can use the session_wise or trial_wise video.
:return:
'''
if session_wise:
input_mmap_file_path = eval(row.loc['alignment_output'])['main']
else:
input_mmap_file_path = eval(row.loc['motion_correction_output'])['main']
#load the mmap file
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
logging.debug(f'{row.name} Loaded movie. dims = {dims}, T = {T}.')
#create a caiman movie with the mmap file
images = Yr.T.reshape((T,) + dims, order='F')
images = cm.movie(images)
#load source extraction result
output = eval(row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
#estimate the background from the extraction
W, b0 = cm.source_extraction.cnmf.initialization.compute_W(Yr, cnm.estimates.A.toarray(), cnm.estimates.C,
cnm.estimates.dims, 1.4 * 5, ssub=2)
cnm.estimates.W = W
cnm.estimates.b0 = b0
# this part could be use with the lastest caiman version
# movie_dir = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/processed/movies/'
# file_name = db.create_file_name(5,row.name)
# cnm.estimates.play_movie(cnm.estimates, images, movie_name= movie_dir + file_name + '.avi')
frame_range = slice(None, None, None)
# create a movie with the model : estimated A and C matrix
Y_rec = cnm.estimates.A.dot(cnm.estimates.C[:, frame_range])
Y_rec = Y_rec.reshape(dims + (-1,), order='F')
Y_rec = Y_rec.transpose([2, 0, 1])
# convert the variable to a caiman movie type
Y_rec = cm.movie(Y_rec)
## this part of the function is a copy from a caiman version
ssub_B = int(round(np.sqrt(np.prod(dims) / W.shape[0])))
B = images[frame_range].reshape((-1, np.prod(dims)), order='F').T - \
cnm.estimates.A.dot(cnm.estimates.C[:, frame_range])
if ssub_B == 1:
B = b0[:, None] + W.dot(B - b0[:, None])
else:
B = b0[:, None] + (np.repeat(np.repeat(W.dot(
downscale(B.reshape(dims + (B.shape[-1],), order='F'),
(ssub_B, ssub_B, 1)).reshape((-1, B.shape[-1]), order='F') -
downscale(b0.reshape(dims, order='F'),
(ssub_B, ssub_B)).reshape((-1, 1), order='F'))
.reshape(
((dims[0] - 1) // ssub_B + 1, (dims[1] - 1) // ssub_B + 1, -1), order='F'),
ssub_B, 0), ssub_B, 1)[:dims[0], :dims[1]].reshape(
(-1, B.shape[-1]), order='F'))
B = B.reshape(dims + (-1,), order='F').transpose([2, 0, 1])
Y_rec_2 = Y_rec + B
Y_res = images[frame_range] - Y_rec - B
images_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2]))
images_np = images[time_cropping[0]:time_cropping[1],:,:]
images_np = images_np / np.max(images_np)
images_np = cm.movie(images_np)
Y_rec_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2]))
Y_rec_np = Y_rec[time_cropping[0]:time_cropping[1],:,:]
Y_rec_np = Y_rec_np / np.max(Y_rec_np)
Y_rec_np = cm.movie(Y_rec_np)
Y_res_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2]))
Y_res_np = Y_res[time_cropping[0]:time_cropping[1],:,:]
Y_res_np = Y_res_np / np.max(Y_res_np)
Y_res_np = cm.movie(Y_res_np)
B_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2]))
B_np = B[time_cropping[0]:time_cropping[1],:,:]
B_np = B_np / np.max(B_np)
B_np = cm.movie(B_np)
mov1 = cm.concatenate((images_np, Y_rec_np), axis=2)
mov2 = cm.concatenate((B_np, Y_res_np), axis=2)
mov = cm.concatenate((mov1, mov2), axis=1)
figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/movies/'
figure_name = db.create_file_name(5,row.name)
#mov.save(figure_path+figure_name+'.tif')
mov.save(figure_path+figure_name+'_'+f'{time_cropping[0]}' + '_' + f'{time_cropping[1]}'+'.tif')
return
mouse_row = selected_rows.iloc[0]
mouse_row_new = main_source_extraction(mouse_row, parameters_source_extraction, dview)
mouse_row_new = db.set_version_analysis('source_extraction', mouse_row_new)
states_df = db.append_to_or_merge_with_states_df(states_df, mouse_row_new)
db.save_analysis_states_database(states_df, path=analysis_states_database_path, backup_path = backup_path)
#%% COMPONENT EVALUATION
min_SNR = 3 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
parameters_component_evaluation = {'min_SNR': min_SNR,
'rval_thr': r_values_min,
'use_cnn': False}
cm.stop_server(dview=dview)
mouse_row_new = main_component_evaluation(mouse_row_new, parameters_component_evaluation)
component_evaluation_output = eval(mouse_row_new['component_evaluation_output'])
input_hdf5_file_path = component_evaluation_output['main']
cnm = load_CNMF(input_hdf5_file_path)
print('Accepted components = ')
print(len(cnm.estimates.idx_components))
print('Rejected components = ')
print(len(cnm.estimates.idx_components_bad))
def signal_to_noise(data_path,
agonia_th,
ground_truth=None,
neurofinder=False):
'''Calculate signal to noise ratio of each box
Parameters
----------
data_path : string
path to folder containing the data
agonia_th : float
threshold for detection confidence for each box'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load Caiman results
cnm = cnmf.load_CNMF(results_caiman_path)
# calculate the centers of the CaImAn factors
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
# keep only cells above confidence threshold
boxes = boxes[boxes[:, 4] > agonia_th].astype('int')
k = 0
for cell, box in enumerate(boxes):
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
if not idx_factor:
boxes = np.delete(boxes, cell - k, axis=0)
k += 1
if ground_truth is not None:
stats = ag.compute_stats(np.array(ground_truth),
boxes,
iou_threshold=0.5)
stnr = [] #np.zeros(boxes.shape[0])
if neurofinder:
for cell, box in enumerate(boxes):
if ground_truth is not None:
if cell in stats['true_positives'][:, 1]:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
box_trace_arr = np.array(images[:, box[1]:box[3],
box[0]:box[2]])
box_trace_arr_cut = box_trace_arr.copy()
box_trace_arr_cut[box_trace < 30] = np.nan
mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2))
trace_flat = box_trace[box_trace > 30]
noise_box = np.std(
trace_flat[trace_flat < np.percentile(trace_flat, 25)])
amp_trace = max(mean_box) - min(mean_box)
stnr.append(amp_trace / noise_box)
else:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
box_trace_arr = np.array(images[:, box[1]:box[3],
box[0]:box[2]])
box_trace_arr_cut = box_trace_arr.copy()
box_trace_arr_cut[box_trace < 30] = np.nan
mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2))
trace_flat = box_trace[box_trace > 30]
noise_box = np.std(
trace_flat[trace_flat < np.percentile(trace_flat, 25)])
amp_trace = max(mean_box) - min(mean_box)
stnr.append(amp_trace / noise_box)
else:
for cell, box in enumerate(boxes):
if ground_truth is not None:
if cell in stats['true_positives'][:, 1]:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
mean_box = box_trace.mean(axis=(1, 2))
amp_trace = max(mean_box) - min(mean_box)
noise_box = np.std(box_trace[box_trace < np.percentile(
box_trace[box_trace != 0], 25)])
stnr.append(amp_trace / noise_box)
else:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
mean_box = box_trace.mean(axis=(1, 2))
amp_trace = max(mean_box) - min(mean_box)
noise_box = np.std(box_trace[
box_trace < np.percentile(box_trace[box_trace != 0], 25)])
stnr.append(amp_trace / noise_box)
return np.array(stnr)
def trace_correlation(data_path,
agonia_th,
select_cells=False,
plot_results=True,
denoise=False):
'''Calculate the correlation between the mean of the Agonia Box and the CaImAn
factor.
Parameters
----------
data_path : string
path to folder containing the data
agonia_th : float
threshold for detection confidence for each box
select_cells: bool, optional
if True get index of active cells
plot_results: bool, optional
if True do boxplot of correlation values for all cells, if selected_cells,
use only active cells
denoise : bool, optional
if True subtract neuropil
Returns
-------
cell_corr : numpy array
corrcoef value for all cells
idx_active : list
if select_cells=True returns index of active cells, otherwise returns
index of all cells
boxes_traces : NxT ndarray
temporal trace for the N box that has an asociated caiman factor'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
# load Caiman results
cnm = cnmf.load_CNMF(results_caiman_path)
# calculate the centers of the CaImAn factors
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
# load boxes
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
# keep only cells above confidence threshold
boxes = boxes[boxes[:, 4] > agonia_th].astype('int')
#delete boxes that do not have a caiman cell inside
k = 0
for cell, box in enumerate(boxes):
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
if not idx_factor:
boxes = np.delete(boxes, cell - k, axis=0)
k += 1
# Load video as 3D tensor (each plane is a frame)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
boxes_traces = np.empty((boxes.shape[0], images.shape[0]))
#calculate correlations between boxes and CaImAn factors
cell_corr = np.empty(len(boxes_traces))
neuropil_trace = np.zeros(T)
if denoise:
_, neuropil_trace, neuropil_power = substract_neuropil(
data_path, agonia_th, 100, 80)
for cell, box in enumerate(boxes):
# calculate boxes traces as means over images
#boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2))-neuropil_trace
boxes_traces[cell] = images[:, box[1]:box[3],
box[0]:box[2]].mean(axis=(1, 2))
#for using the percentile criteria
med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0)
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
boxes_traces[cell] = box_trace[:,
np.logical_and(
med > np.percentile(med, 80),
med < np.percentile(med, 95))].mean(
axis=1)
boxes_traces[
cell] = boxes_traces[cell] - neuropil_trace * neuropil_power * .7
#boxes_traces[cell] = boxes_traces[cell]-neuropil_trace*boxes_traces[cell].mean()
# get the asociated CaImAn factor by checking if its center of mass is inside the box
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
# in case there is more than one center inside the box choose the one closer to the center of the box
if len(idx_factor) > 1:
idx_factor = [
idx_factor[np.argmin([
np.linalg.norm([(box[3] - box[1]) / 2, (box[2] - box[0]) /
2] - c) for c in centers[idx_factor]
])]
]
cell_corr[cell] = np.corrcoef(
[cnm.estimates.C[idx_factor[0]], boxes_traces[cell]])[1, 0]
if select_cells:
#select only active cells using CaImAn criteria
fitness, _, _, _ = compute_event_exceptionality(boxes_traces)
idx_active = [cell for cell, fit in enumerate(fitness) if fit < -20]
else:
idx_active = [cell for cell, _ in enumerate(boxes_traces)]
if plot_results:
corr_toplot = [
corr for id, corr in enumerate(cell_corr)
if id in idx_active and ~np.isnan(corr)
]
fig, ax = plt.subplots()
p = ax.boxplot(corr_toplot)
ax.set_ylim([-1, 1])
plt.text(1.2, 0.8, 'n_cells = {}'.format(len(corr_toplot)))
return cell_corr, idx_active, boxes_traces
'block_size_spat': block_size,
'num_blocks_per_run_spat': num_blocks_per_run,
'num_blocks_per_run_temp': num_blocks_per_run,
'n_pixels_per_process': n_pixels_per_process,
'ssub': 2,
'tsub': 2,
'p_tsub': 1,
'p_ssub': 1,
'thr_method': 'nrg'
}
init_method = global_params['init_method']
opts = params.CNMFParams(params_dict=params_dict)
cnm = load_CNMF(fname_new[:-5] + '_cnmf_gsig.hdf5')
# %% prepare ground truth masks
gt_file = os.path.join(
os.path.split(fname_new)[0],
os.path.split(fname_new)[1][:-4] + 'match_masks.npz')
with np.load(gt_file, encoding='latin1') as ld:
print(ld.keys())
Cn_orig = ld['Cn']
gt_estimate = Estimates(A=scipy.sparse.csc_matrix(ld['A_gt'][()]),
b=ld['b_gt'],
C=ld['C_gt'],
f=ld['f_gt'],
R=ld['YrA_gt'],
dims=(ld['d1'], ld['d2']))
max_ = img.max()
else:
min_, max_ = min_max
img = (img - min_) / (max_ - min_) * gain
img = img.astype(out_type)
img = np.dstack([img] * 3)
return img
F = FileDialog()
# load object saved by CNMF
fpath = F.getOpenFileName(caption='Load CNMF Object',
filter='HDF5 (*.h5 *.hdf5);;NWB (*.nwb)')[0]
cnm_obj = load_CNMF(fpath)
# movie
if not os.path.exists(cnm_obj.mmap_file):
M = FileDialog()
cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file',
filter='*.mmap')[0]
if fpath[-3:] == 'nwb':
mov = cm.load(cnm_obj.mmap_file,
var_name_hdf5='acquisition/TwoPhotonSeries')
else:
mov = cm.load(cnm_obj.mmap_file)
estimates = cnm_obj.estimates
params_obj = cnm_obj.params
def substract_neuropil(data_path, agonia_th, neuropil_pctl, signal_pctl):
'''Calculate the neuropil trace as the average fluorescence of every pixel that is not in a box.
subtract it to every box trace, calculated as the activity of the pixels above the signal_pctl.
Parameters
----------
data_path : string
path to folder containing the data
agonia_th : float
threshold for detection confidence for each box
neuropil_pctl : int
percentile of pixels intensity bellow which pixel is considered to calculate neuropil
signal_pctl : int
percentile of pixels intensity above which pixel is considered to calculate trace
Returns
-------
denoised_traces : NxT ndarray
temporal traces after neuropil subtraction for the N boxes
neuropil_trace : array
normalized temporal trace of background noise, length T
neuropil_power : float
mean of neuropil trace before normalization
'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load Caiman results
cnm = cnmf.load_CNMF(results_caiman_path)
# calculate the centers of the CaImAn factors
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
# keep only cells above confidence threshold
boxes = boxes[boxes[:, 4] > agonia_th].astype('int')
#delete boxes that do not have a caiman cell inside
k = 0
for cell, box in enumerate(boxes):
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
if not idx_factor:
boxes = np.delete(boxes, cell - k, axis=0)
k += 1
boxes_traces = np.empty((boxes.shape[0], images.shape[0]))
mask = np.zeros(images.shape[1:], dtype=bool)
for cell, box in enumerate(boxes):
#boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2))
med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0)
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
boxes_traces[cell] = box_trace[:,
med > np.percentile(med, signal_pctl
)].mean(axis=1)
mask[box[1].astype('int'):box[3].astype('int'),
box[0].astype('int'):box[2].astype('int')] = 1
mask = (1 - mask).astype('bool')
not_cell = images[:, mask]
#not_cell_med = np.median(not_cell,axis=0)
#neuropil_trace = not_cell[:,not_cell_med<np.percentile(not_cell_med,neuropil_pctl)].mean(axis=1)
#denoised_traces = boxes_traces-neuropil_trace
neuropil_trace = not_cell.mean(axis=1)
neuropil_power = neuropil_trace.mean()
neuropil_trace = neuropil_trace / neuropil_power
denoised_traces = np.array([
boxes_traces[i] - neuropil_trace * boxes_traces[i].mean()
for i in range(boxes_traces.shape[0])
])
return denoised_traces, neuropil_trace, neuropil_power
