def plot_traces_multiple_evaluated(row = None, session_wise = False): ''' Plots different versions of contour images that change the inicialization parameters for source extraccion. The idea is to see the impact of different seed selection in the final source extraction result. :param row: one analysis state row :return: figure ''' corr_min = round(eval(row['source_extraction_parameters'])['min_corr'],1) pnr_min = round(eval(row['source_extraction_parameters'])['min_pnr'],1) r_min = eval(row['component_evaluation_parameters'])['rval_thr'] snf_min = eval(row['component_evaluation_parameters'])['min_SNR'] output_source_extraction = eval(row.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) C = cnm.estimates.C output_component_evaluation = eval(row.loc['component_evaluation_output']) cnm_file_path = output_component_evaluation['main'] cnm_eval = load_CNMF(db.get_file(cnm_file_path)) idx = cnm_eval.estimates.idx_components idx_b = cnm_eval.estimates.idx_components_bad fig, ax = plt.subplots(1) C[0] += C[0].min() for i in range(1, len(C)): C[i] += C[i].min() + C[:i].max() if i in idx_b: color = 'red' else: color = 'blue' ax.plot(C[i],color = color) ax.set_xlabel('t [frames]') ax.set_yticks([]) ax.set_ylabel('activity') ax.set_title('Corr=' + f'{corr_min}' + ', PNR = ' + f'{pnr_min}' + ', PCC = ' + f'{r_min}' + ', SNR =' + f'{snf_min}') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/component_evaluation/trial_wise/meta/figures/traces/' if session_wise: fig_dir = 'data/interim/component_evaluation/session_wise/meta/figures/traces/' fig_name = fig_dir + db.create_file_name(5,row.name) + '.png' fig.savefig(fig_name) return
def plot_contours_evaluated(row = None, session_wise = False): ''' Plot contours for all cells, selected cells and non selected cells, and saves it in figure_path = '/data/interim/component_evaluation/trial_wise/meta/figures/contours/' :param row: one analysis state row ''' index = row.name corr_min = round(eval(row['source_extraction_parameters'])['min_corr'],1) pnr_min = round(eval(row['source_extraction_parameters'])['min_pnr'],1) r_min = eval(row['component_evaluation_parameters'])['rval_thr'] snf_min = eval(row['component_evaluation_parameters'])['min_SNR'] output_source_extraction = eval(row.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) output_component_evaluation = eval(row.loc['component_evaluation_output']) cnm_file_path = output_component_evaluation['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) figure, axes = plt.subplots(1, 3) axes[0].imshow(cn_filter) axes[1].imshow(cn_filter) axes[2].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[0].plot(*v.T, c='w') axes[0].set_title('All components') axes[0].set_ylabel('Corr=' + f'{corr_min}' + ', PNR = ' + f'{pnr_min}' + ', PCC = ' + f'{r_min}' + ', SNR =' + f'{snf_min}') idx = cnm.estimates.idx_components coordinates = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx], np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[1].plot(*v.T, c='b') axes[1].set_title('Accepted components') idx_b = cnm.estimates.idx_components_bad coordinates_b = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx_b], np.shape(cn_filter), 0.2, 'max') for c in coordinates_b: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[2].plot(*v.T, c='r') axes[2].set_title('Rejected components') figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/trial_wise/meta/figures/contours/' if session_wise: figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/component_evaluation/session_wise/meta/figures/contours/' figure_name = figure_path + db.create_file_name(5,index) + '.png' figure.savefig(figure_name) return figure
def get_fig_session_wise_neurons_per_trial_histogram(mouse, session): # Get selected rows selected_rows = src.pipeline.select('component_evaluation', mouse, session) nr_components_list = [] # Get the number of components for each trial for index, row in selected_rows.iterrows(): try: cnm = load_CNMF( src.pipeline.get_file( eval(row.loc['source_extraction_output'])['main'])) nr_components_list.append(len(cnm.estimates.C)) except: pass # Number of trials N = len(nr_components_list) fig, ax = plt.subplots(1) plt.hist(nr_components_list) plt.ylabel('# trials') plt.xlabel('K') print('N = ', N) return fig
def get_fig_session_wise_neurons_per_trial(mouse, session): # Get selected rows selected_rows = src.pipeline.select('component_evaluation', mouse, session) trial_string_list = [] nr_components_list = [] # Get the number of components for each trial for index, row in selected_rows.iterrows(): mouse, session, trial, is_rest, *r = index trial_string = str(trial) + '_R' if bool(is_rest) else str(trial) trial_string_list.append(trial_string) if type(row.loc['source_extraction_output']) is str: cnm_file_path = eval(row.loc['source_extraction_output'])['main'] else: cnm_file_path = f'data/interim/source_extraction/trial_wise/main/{src.pipeline.create_file_name(step_index, index)}.hdf5' cnm = load_CNMF(cnm_file_path) nr_components_list.append(len(cnm.estimates.C)) # Number of trials N = len(nr_components_list) fig, ax = plt.subplots(1) # fig.size(N * 50, 500) x_pos = np.arange(N) plt.bar(x_pos, nr_components_list) plt.ylabel('# components') plt.xticks(x_pos, trial_string_list) print('N = ', N) return fig
def load_results(results_dir): # Determine filename from dir fn = os.path.join(results_dir, 'analysis_results.hdf5') # Load CNMF object using CaImAn function cnm = cnmf.load_CNMF(fn) print('Number of components: {}'.format(cnm.estimates.C.shape[0])) return cnm
def plot_traces_multiple(row, version=None, corr_array=None, pnr_array=None, session_wise=False): ''' Plots different versions of contour images that change the inicialization parameters for source extraccion. The idea is to see the impact of different seed selection in the final source extraction result. :param row: one analysis state row :param version: array containing the version numbers of source extraction that will be ploted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :param: session_wise bool that indicates where the figure is save :return: None ''' states_df = db.open_analysis_states_database() index = row.name for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): fig, ax = plt.subplots(1) new_row = db.select( states_df, 'component_evaluation', mouse=index[0], session=index[1], trial=index[2], is_rest=index[3], cropping_v=index[5], motion_correction_v=index[6], alignment_v=index[7], source_extraction_v=version[ii * len(pnr_array) + jj]) new_row = new_row.iloc[0] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) C = cnm.estimates.C idx_components = cnm.estimates.idx_components C[0] += C[0].min() for i in range(1, len(C)): C[i] += C[i].min() + C[:i].max() ax.plot(C[i]) ax.set_xlabel('t [frames]') ax.set_yticks([]) ax.set_ylabel('activity') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/source_extraction/trial_wise/meta/figures/traces/' if session_wise: fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/traces/' fig_name = fig_dir + db.create_file_name( 3, new_row.name ) + '_corr_min' + f'{round(corr_array[ii], 1)}' + '_pnr_min' + f'{round(pnr_array[jj], 1)}' + '_.png' fig.savefig(fig_name) return
def plot_session_contours(selected_rows, version=None, corr_array=None, pnr_array=None): ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param selected_rows: rows corresponding to different trials :param version: array containing the version numbers of source extraction that will be plotted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :return: (saves multiple figures) ''' states_df = db.open_analysis_states_database() for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): figure, axes = plt.subplots(1, len(selected_rows), figsize=(50, 10)) for i in range(len(selected_rows)): row = selected_rows.iloc[i] index = row.name new_row = db.select(states_df, 'component_evaluation', mouse=index[0], session=index[1], trial=index[2], is_rest=index[3], cropping_v=index[5], motion_correction_v=index[6], alignment_v=index[7], source_extraction_v=version[ii * len(pnr_array) + jj]) new_row = new_row.iloc[0] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) # axes[i].imshow(np.clip(cn_filter, min_corr, max_corr), cmap='viridis') axes[i].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[i].plot(*v.T, c='w') axes[i].set_title('Trial = ' + f'{i + 1}', fontsize=30) axes[i].set_xlabel('#cells = ' + f'{cnm.estimates.A.shape[1]}', fontsize=30) figure.suptitle('min_corr = ' + f'{round(corr_array[ii], 2)}' + 'min_pnr = ' + f'{round(pnr_array[jj], 2)}', fontsize=50) fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, new_row.name) + '_version_' + f'{version[ii * len(pnr_array) + jj]}' + '.png' figure.savefig(fig_name) return
def plot_source_extraction_result(mouse_row_new): ''' Generates and saves a contour plot and a trace plot for the specific mouse_row ''' corr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_corr'], 1) pnr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_pnr'], 1) output_source_extraction = eval(mouse_row_new.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) figure, axes = plt.subplots(1) axes.imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes.plot(*v.T, c='w') axes.set_title('min_corr = ' + f'{corr_min}') axes.set_ylabel('min_pnr = ' + f'{pnr_min}') fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' file_name = db.create_file_name(3, mouse_row_new.name) figure.savefig(fig_dir + file_name + '.png') ## up to here fig, ax = plt.subplots(1) C = cnm.estimates.C C[0] += C[0].min() for i in range(1, len(C)): C[i] += C[i].min() + C[:i].max() ax.plot(C[i]) ax.set_xlabel('t [frames]') ax.set_yticks([]) ax.set_ylabel('activity') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/traces/' fig_name = fig_dir + db.create_file_name(3, mouse_row_new.name) + '.png' fig.savefig(fig_name) return
def plot_source_extraction_result_specific_cell(mouse_row_new, cell_number): ''' (Still need to be finished) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! THERE IS AN ERROR IN THE In the first plot shows correlation image and contour of the selected neurons. In the second plot shows the traces for the selected neurons. :param mouse_row_new: data base row :param cell_number: array with the cells that are selected to be ploted :return: None ''' corr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_corr'], 1) pnr_min = round(eval(mouse_row_new['source_extraction_parameters'])['min_pnr'], 1) output_source_extraction = eval(mouse_row_new.loc['source_extraction_output']) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) f, (a0, a1) = plt.subplots(2, 1, gridspec_kw={'height_ratios': [3, 1]}) a0.imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for i in cell_number: v = coordinates[i]['coordinates'] coordinates[i]['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] a0.plot(*v.T, c='w') a0.set_title('Contour Plot') fig, ax = plt.subplots(1) C = cnm.estimates.C C[0] += C[0].min() for i in range(cell_number): C[i] += C[i].min() + C[:i].max() a1.plot(C[i]) a1.set_xlabel('t [frames]') a1.set_yticks([]) a1.set_title('Calcium Traces') fig.set_size_inches([10., .3 * len(C)]) fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/' fig_name = fig_dir + db.create_file_name(3, mouse_row_new.name) + '_example.png' f.savefig(fig_name) return
def plot_multiple_contours_session_wise(selected_rows, version = None , corr_array = None, pnr_array = None): ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param selected_rows: all analysis state selected :param version: array containing the version numbers of source extraction that will be plotted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :return: figure ''' states_df = db.open_analysis_states_database() figure, axes = plt.subplots(len(corr_array), len(pnr_array), figsize=(15, 15)) color = ['w','b','r','m','c'] for row in range(len(selected_rows)): mouse_row = selected_rows.iloc[row] index = mouse_row.name output = eval(mouse_row.loc['source_extraction_output']) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): axes[ii, jj].imshow(cn_filter) new_row = db.select(states_df, 'component_evaluation', mouse=index[0], session=index[1], trial=index[2], is_rest=index[3], cropping_v=index[5], motion_correction_v=index[6], source_extraction_v=version[ii * len(pnr_array) + jj]) new_row = new_row.iloc[0] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[ii, jj].plot(*v.T, c = color[row]) axes[ii, jj].set_title('min_corr = ' + f'{round(corr_array[ii],2)}') axes[ii, jj].set_ylabel('min_pnr = ' + f'{round(pnr_array[jj],2)}') fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, new_row.name)+'_corr_min' + f'{round(corr_array[0],1)}'+ '_pnr_min'+f'{round(pnr_array[0],1)}' + '_all.png' figure.savefig(fig_name) return figure
def plot_multiple_contours(rows, version = None , corr_array = None, pnr_array = None,session_wise = False): ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param row: one analysis state row :param version: array containing the version numbers of source extraction that will be plotted :param corr_array: array of the same length of version and pnr_array containing the min_corr values for those versions :param pnr_array: array of the same length of version and corr_array containing the min_pnr values for those versions :return: figure ''' figure, axes = plt.subplots(len(corr_array), len(pnr_array), figsize=(15, 15)) for ii in range(corr_array.shape[0]): for jj in range(pnr_array.shape[0]): version_number = ii *corr_array.shape[0] + jj + 1 if version_number in version: new_row = rows.query('(source_extraction_v == ' + f'{version_number}' + ')') new_row = new_row.iloc[0] output = eval(new_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) axes[ii, jj].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[ii, jj].plot(*v.T, c='w') axes[ii, jj].set_title('min_corr = ' + f'{round(corr_array[ii],2)}') axes[ii, jj].set_ylabel('min_pnr = ' + f'{round(pnr_array[jj],2)}') fig_dir = 'data/interim/source_extraction/trial_wise/meta/figures/contours/' if session_wise: fig_dir = 'data/interim/source_extraction/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, new_row.name)+'_corr_min' + f'{round(corr_array[0],1)}'+ '_pnr_min'+f'{round(pnr_array[0],1)}' + '_.png' figure.savefig(fig_name) return figure
def traces_extraction_AGONIA(data_path, agonia_th, neuropil_contaminated=False): '''Extract traces of boxes using agonia Extractor class''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load Caiman results cnm = cnmf.load_CNMF(results_caiman_path) # calculate the centers of the CaImAn factors centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() # keep only cells above confidence threshold boxes = boxes[boxes[:, 4] > agonia_th].astype('int') k = 0 for cell, box in enumerate(boxes): idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] if not idx_factor: boxes = np.delete(boxes, cell - k, axis=0) k += 1 ola = Extractor() for frame in images: ola.extract(frame, boxes) traces, traces_with_bg = ola.get_traces() ola.pool.terminate() if neuropil_contaminated: return traces.T, traces_with_bg.T else: return traces.T
def boxes_exploration_interactive(data_path): '''Returns an interactive plot with the agonia boxes with a confidence value above a Score selected by the user with a slider. The user can select a box by clicking on it and mean box Fluorescence and the Caiman DF/F of such box will be ploted.''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) cnm = cnmf.load_CNMF(results_caiman_path) img = hv.Image(median_projection, bounds=(0, 0, median_projection.shape[1], median_projection.shape[0])).options(cmap='gray') with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) #scatter = hv.Scatter((centers[:,1], median_projection.shape[0] - centers[:,0])) kdims = [hv.Dimension('Score', values=np.arange(0.05, 1, 0.05))] tap = streams.SingleTap(transient=True, source=img) Experiment = Stream.define('Experiment', data_path=data_path) Centers = Stream.define('Centers', centers=centers) CaImAn_detection = Stream.define('CaImAn_detection', cnm=cnm) dmap = hv.DynamicMap(plot_AGonia_boxes_interactive, kdims=kdims, streams=[Experiment(), tap]) dmap1 = hv.DynamicMap(plot_boxes_traces, kdims=kdims, streams=[Experiment(), tap]) dmap2 = hv.DynamicMap( plot_seeded_traces, kdims=kdims, streams=[CaImAn_detection(), Experiment(), tap, Centers()]) return ((img * dmap).opts(width=500, height=500) + dmap1.opts(width=500, height=250) + dmap2.opts(width=500, height=250)).opts( opts.Curve(framewise=True)).cols(1)
def load_hdf5_result(result_dir, gdrive_result_dir, rclone_config, use_filtered=False): hdf5_filename = 'analysis_results.hdf5' if use_filtered: hdf5_filename = 'analysis_results_filtered.hdf5' result_dir = os.path.join(result_dir, 'filtered') gdrive_result_dir = os.path.join(gdrive_result_dir, 'filtered') h5fpath = os.path.join(result_dir, hdf5_filename) if not os.path.isfile(h5fpath): success = gdrive_download_file( os.path.join(gdrive_result_dir, hdf5_filename), result_dir, rclone_config) if not success: return None from caiman.source_extraction.cnmf.cnmf import load_CNMF try: return load_CNMF(h5fpath) except OSError as e: logging.error('Failed to open file: %s', h5fpath) raise e
if not os.path.exists(outDir): os.mkdir(outDir) if not os.path.exists(outImageFolder): os.mkdir(outImageFolder) print('Input hdf5 file: {}'.format(hdf5File), flush=True) print('Output directory: {}'.format(outDir), flush=True) # load in data print('Loading data', flush=True) if args.online: cnm = cnmf.online_cnmf.load_OnlineCNMF(hdf5File) else: cnm = cnmf.load_CNMF(hdf5File) cnm.estimates.dims = cnm.dims A, C, b, f = tp.cellInfoCaimanHdf5(hdf5File, dims=cnm.dims) print('Calculating masks and stats. nSources: {}, maskThresh: {}'.format( nSources, maskThresh), flush=True) meanImageSources = np.tensordot(C.mean(axis=0), A, axes=(0, 0)).transpose().astype('float32') meanImage = meanImageSources + np.tensordot( f.mean(axis=0), b, axes=(0, 0)).transpose().astype('float32') stdImageSources = np.tensordot(C.std(axis=0), A,
# %% imports %matplotlib qt5 import numpy as np import matplotlib.pyplot as plt from matplotlib.animation import FuncAnimation import caiman as cm from caiman.source_extraction.cnmf import cnmf as cnmf from pathlib import Path # %% load footprints # path = "/media/georg/data/mesoscope/first data/with behavior/2021-08-24_day1_overview/reshaped/memmap__d1_1128_d2_1068_d3_1_order_C_frames_3901_sel.hdf5" path = "/media/georg/data/mesoscope/first data/with behavior/2021-08-25_day2_square_1/reshaped/memmap__d1_1892_d2_2504_d3_1_order_C_frames_850_sel.hdf5" cnmf2 = cnmf.load_CNMF(path) # %% infer stuff p = Path(cnmf2.mmap_file).stem.split('_') xpx = int(p[p.index('d1')+1]) ypx = int(p[p.index('d2')+1]) nFrames = int(p[p.index('frames')+1]) # with open(folder.parent / 'meta.json','r') as fH: # meta = json.load(fH) # or set for now fs = 3.03 # %% cnmf2.estimates.detrend_df_f(quantileMin=8,frames_window=100) D = cnmf2.estimates.F_dff D_orig = copy(D) D = D[good_inds,:]
'check_nan': check_nan, 'block_size_temp': block_size, 'block_size_spat': block_size, 'num_blocks_per_run_spat': num_blocks_per_run, 'num_blocks_per_run_temp': num_blocks_per_run, 'n_pixels_per_process': n_pixels_per_process, 'ssub': global_params['ssub'], 'tsub': global_params['tsub'], 'thr_method': 'nrg' } init_method = global_params['init_method'] opts = params.CNMFParams(params_dict=params_dict) if reload: cnm2 = load_CNMF(fname_new[:-5] + '_cnmf_perf_web.hdf5') reproduce_images_website = False if reproduce_images_website: cnm2.estimates.evaluate_components(images, params=cnm2.params, dview=dview) pl.figure(figsize=(10, 7)) good_snr = np.where(cnm2.estimates.SNR_comp > 5)[0] examples = np.argsort(cnm2.estimates.cnn_preds)[-100:] examples = np.intersect1d(examples, good_snr)[-5:] for count, ex in enumerate(examples): pl.subplot(6, 2, 2 * count + 1) img = cnm2.estimates.A[:, ex].toarray().reshape( cnm2.estimates.dims, order='F') cm_ = np.array( scipy.ndimage.measurements.center_of_mass(img)).astype(
def plot_multiple_contours_session_wise_evaluated(selected_rows): ## IN DEVELOPMENT!!!!!!! ''' Plots different versions of contour images that change the initialization parameters for source extraction. The idea is to see the impact of different seed selection in the final source extraction result. :param selected_rows: all analysis state selected :return: figure ''' figure, axes = plt.subplots(3, 5, figsize=(50, 30)) for row in range(len(selected_rows)): mouse_row = selected_rows.iloc[row] index = mouse_row.name output = eval(mouse_row.loc['source_extraction_output']) corr_path = output['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) axes[0,row].imshow(cn_filter) axes[1,row].imshow(cn_filter) axes[2,row].imshow(cn_filter) output = eval(mouse_row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) coordinates = cm.utils.visualization.get_contours(cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[0,row].plot(*v.T, c = 'w', linewidth=3) axes[0,row].set_title('Trial = ' + f'{row}') axes[0,row].set_ylabel('') output = eval(mouse_row.loc['component_evaluation_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) idx = cnm.estimates.idx_components coordinates = cm.utils.visualization.get_contours(cnm.estimates.A[:, idx], np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[1,row].plot(*v.T, c='b', linewidth=3) idx_b = cnm.estimates.idx_components_bad coordinates_b = cm.utils.visualization.get_contours(cnm.estimates.A[:,idx_b], np.shape(cn_filter), 0.2, 'max') for c in coordinates_b: v = c['coordinates'] c['bbox'] = [np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0]))] axes[2,row].plot(*v.T, c='r', linewidth=3) source_extraction_parameters = eval(mouse_row['source_extraction_parameters']) corr_lim = source_extraction_parameters['min_corr'] pnr_lim = source_extraction_parameters['min_pnr'] component_evaluation_parameters = eval(mouse_row['component_evaluation_parameters']) pcc = component_evaluation_parameters['rval_thr'] SNR = component_evaluation_parameters['min_SNR'] figure.suptitle('Corr = ' + f'{corr_lim}' + 'PNR = ' + f'{pnr_lim}' + 'PCC = ' + f'{pcc}' + 'SNR = ' + f'{SNR}', fontsize=50) fig_dir = 'data/interim/component_evaluation/session_wise/meta/figures/contours/' fig_name = fig_dir + db.create_file_name(3, index)+'_Corr = ' + f'{corr_lim}' + '_PNR = ' + f'{pnr_lim}' + '_PCC = ' + f'{pcc}' + '_SNR = ' + f'{SNR}' +'_.png' figure.savefig(fig_name) return figure
'cnn_lowest': 0.05 } registration_params = {'max_thr': 0.45, 'thresh_cost': 0.75, 'max_dist': 10} """ Prepare data """ gdrive_dated_dir = os.path.join(gdrive_subdir, exp_month, exp_title, exp_date) gdrive_result_dir = os.path.join(gdrive_dated_dir, 'caiman', animal_name) caiman_result_dir = os.path.join(local_miniscope_path, exp_title, exp_date, 'caiman', animal_name) # Download result files if not stored locally h5fpath = os.path.join(caiman_result_dir, 'analysis_results.hdf5') if not os.path.isfile(h5fpath): gdrive_download_file(gdrive_result_dir + '/analysis_results.hdf5', caiman_result_dir, rclone_config) cnm_obj = load_CNMF(h5fpath) def filter_components(cnm_obj, components_quality_params, registration_params): cnm_obj.estimates.threshold_spatial_components( maxthr=registration_params['max_thr']) cnm_obj.estimates.remove_small_large_neurons( min_size_neuro=neuron_size_params['min'], max_size_neuro=neuron_size_params['max']) logging.info('filtered %d neurons based on the size', len(cnm_obj.estimates.idx_components_bad)) cnm_obj.estimates.accepted_list = cnm_obj.estimates.idx_components empty_images = np.zeros((1, ) + cnm_obj.dims) if components_quality_params['use_cnn'] and ( not hasattr(cnm_obj.estimates, 'cnn_preds') or len(cnm_obj.estimates.cnn_preds) == 0):
motion_correction_v=motion_correction_version, alignment_v=0, source_extraction_v=1) row = selected_rows.iloc[1] #input_mmap_file_path = eval(row.loc['alignment_output'])['main'] input_mmap_file_path = eval(row.loc['motion_correction_output'])['main'] Yr, dims, T = cm.load_memmap(input_mmap_file_path) # logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.') images = Yr.T.reshape((T, ) + dims, order='F') images = cm.movie(images) output = eval(row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) W, b0 = cm.source_extraction.cnmf.initialization.compute_W( Yr, cnm.estimates.A.toarray(), cnm.estimates.C, cnm.estimates.dims, 1.4 * 5, ssub=2) cnm.estimates.W = W cnm.estimates.b0 = b0 #movie_dir = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/processed/movies/' #file_name = db.create_file_name(5,row.name) #cnm.estimates.play_movie(cnm.estimates, images, movie_name= movie_dir + file_name + '.avi') frame_range = slice(None, None, None) Y_rec = cnm.estimates.A.dot(cnm.estimates.C[:, frame_range])
_x - x_, fill=False, color='r', linestyle='--', linewidth=3) axes[kk, 0].add_patch(rect) output_cropping = mouse_row['cropping_output'] cropped_file = eval(output_cropping)['main'] m = cm.load(cropped_file) axes[kk, 1].imshow(m[0, :, :], cmap='gray') output_source_extraction = eval( mouse_row['source_extraction_output']) cnm_file_path = output_source_extraction['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) corr_path = output_source_extraction['meta']['corr']['main'] cn_filter = np.load(db.get_file(corr_path)) axes[kk, 2].imshow(cn_filter) coordinates = cm.utils.visualization.get_contours( cnm.estimates.A, np.shape(cn_filter), 0.2, 'max') for c in coordinates: v = c['coordinates'] c['bbox'] = [ np.floor(np.nanmin(v[:, 1])), np.ceil(np.nanmax(v[:, 1])), np.floor(np.nanmin(v[:, 0])), np.ceil(np.nanmax(v[:, 0])) ] axes[kk, 2].plot(*v.T, c='w', linewidth=3)
def run_component_evaluation(input_file, session_wise=False, equalization=False): sql = "SELECT source_extraction_session_wise,min_SNR,alignment_main,equalization_main,motion_correction_main,rval_thr,use_cnn FROM Analysis WHERE source_extraction_main=?" val = [ input_file, ] mycursor.execute(sql, val) myresult = mycursor.fetchall() data = [] aux = [] for x in myresult: aux = x for y in aux: data.append(y) if session_wise: input_mmap_file_path = data[2] elif equalization: input_mmap_file_path = data[3] else: input_mmap_file_path = data[4] parameters = {'min_SNR': data[1], 'rval_thr': data[5], 'use_cnn': data[6]} data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/session_wise/' if \ data[0] else os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/trial_wise/' sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,equalization_v,source_extraction_v,component_evaluation_v,input,home_path,decoding_main FROM Analysis WHERE source_extraction_main=?" val = [ input_file, ] mycursor.execute(sql, val) result = mycursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[10] == 0: data[10] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}" output_file_path = f'main/{file_name}.hdf5' sql1 = "UPDATE Analysis SET component_evaluation_main=?,component_evaluation_v=? WHERE source_extraction_main=? " val1 = [output_file_path, data[10], input_file] mycursor.execute(sql1, val1) else: data[10] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}" output_file_path = f'main/{file_name}.hdf5' sql2 = "INSERT INTO Analysis (component_evaluation_main,component_evaluation_v) VALUES (?,?)" val2 = [output_file_path, data[10]] mycursor.execute(sql2, val2) database.commit() output_file_path_full = data_dir + output_file_path # Load CNMF object cnm = load_CNMF(input_mmap_file_path) # Load the original movie Yr, dims, T = cm.load_memmap(input_mmap_file_path) images = Yr.T.reshape((T, ) + dims, order='F') # Set the parmeters cnm.params.set('quality', parameters) # Stop the cluster if one exists n_processes = psutil.cpu_count() try: cm.cluster.stop_server() except: pass # Start a new cluster c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_processes, # number of process to use, if you go out of memory try to reduce this one single_thread=False) # Evaluate components cnm.estimates.evaluate_components(images, cnm.params, dview=dview) logging.debug('Number of total components: ', len(cnm.estimates.C)) logging.debug('Number of accepted components: ', len(cnm.estimates.idx_components)) # Stop the cluster dview.terminate() # Save CNMF object cnm.save(output_file_path_full) return output_file_path
if min_max is None: min_ = img.min() max_ = img.max() else: min_, max_ = min_max img = (img-min_)/(max_-min_)*gain img = img.astype(out_type) img = np.dstack([img]*3) return img F = FileDialog() # load object saved by CNMF CNMF_filename = F.getOpenFileName(caption='Load CNMF Object',filter='*.*5')[0] cnm_obj = load_CNMF(CNMF_filename) # load movie if not os.path.exists(cnm_obj.mmap_file): M = FileDialog() cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file', filter='*.mmap')[0] mov = cm.load(cnm_obj.mmap_file) min_mov = np.min(mov) max_mov = np.max(mov) # load summary image Cn = cnm_obj.estimates.Cn estimates = cnm_obj.estimates
def create_video(row, time_cropping, session_wise = False): ''' This fuction creates a complete video with raw movie (motion corrected), source extracted cells and source extraction + background. :param row: pandas dataframe containing the desired processing information to create the video. It can use the session_wise or trial_wise video. :return: ''' if session_wise: input_mmap_file_path = eval(row.loc['alignment_output'])['main'] else: input_mmap_file_path = eval(row.loc['motion_correction_output'])['main'] #load the mmap file Yr, dims, T = cm.load_memmap(input_mmap_file_path) logging.debug(f'{row.name} Loaded movie. dims = {dims}, T = {T}.') #create a caiman movie with the mmap file images = Yr.T.reshape((T,) + dims, order='F') images = cm.movie(images) #load source extraction result output = eval(row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) #estimate the background from the extraction W, b0 = cm.source_extraction.cnmf.initialization.compute_W(Yr, cnm.estimates.A.toarray(), cnm.estimates.C, cnm.estimates.dims, 1.4 * 5, ssub=2) cnm.estimates.W = W cnm.estimates.b0 = b0 # this part could be use with the lastest caiman version # movie_dir = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/processed/movies/' # file_name = db.create_file_name(5,row.name) # cnm.estimates.play_movie(cnm.estimates, images, movie_name= movie_dir + file_name + '.avi') frame_range = slice(None, None, None) # create a movie with the model : estimated A and C matrix Y_rec = cnm.estimates.A.dot(cnm.estimates.C[:, frame_range]) Y_rec = Y_rec.reshape(dims + (-1,), order='F') Y_rec = Y_rec.transpose([2, 0, 1]) # convert the variable to a caiman movie type Y_rec = cm.movie(Y_rec) ## this part of the function is a copy from a caiman version ssub_B = int(round(np.sqrt(np.prod(dims) / W.shape[0]))) B = images[frame_range].reshape((-1, np.prod(dims)), order='F').T - \ cnm.estimates.A.dot(cnm.estimates.C[:, frame_range]) if ssub_B == 1: B = b0[:, None] + W.dot(B - b0[:, None]) else: B = b0[:, None] + (np.repeat(np.repeat(W.dot( downscale(B.reshape(dims + (B.shape[-1],), order='F'), (ssub_B, ssub_B, 1)).reshape((-1, B.shape[-1]), order='F') - downscale(b0.reshape(dims, order='F'), (ssub_B, ssub_B)).reshape((-1, 1), order='F')) .reshape( ((dims[0] - 1) // ssub_B + 1, (dims[1] - 1) // ssub_B + 1, -1), order='F'), ssub_B, 0), ssub_B, 1)[:dims[0], :dims[1]].reshape( (-1, B.shape[-1]), order='F')) B = B.reshape(dims + (-1,), order='F').transpose([2, 0, 1]) Y_rec_2 = Y_rec + B Y_res = images[frame_range] - Y_rec - B images_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) images_np = images[time_cropping[0]:time_cropping[1],:,:] images_np = images_np / np.max(images_np) images_np = cm.movie(images_np) Y_rec_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) Y_rec_np = Y_rec[time_cropping[0]:time_cropping[1],:,:] Y_rec_np = Y_rec_np / np.max(Y_rec_np) Y_rec_np = cm.movie(Y_rec_np) Y_res_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) Y_res_np = Y_res[time_cropping[0]:time_cropping[1],:,:] Y_res_np = Y_res_np / np.max(Y_res_np) Y_res_np = cm.movie(Y_res_np) B_np = np.zeros((time_cropping[1]-time_cropping[0],images.shape[1],images.shape[2])) B_np = B[time_cropping[0]:time_cropping[1],:,:] B_np = B_np / np.max(B_np) B_np = cm.movie(B_np) mov1 = cm.concatenate((images_np, Y_rec_np), axis=2) mov2 = cm.concatenate((B_np, Y_res_np), axis=2) mov = cm.concatenate((mov1, mov2), axis=1) figure_path = '/home/sebastian/Documents/Melisa/calcium_imaging_analysis/data/interim/movies/' figure_name = db.create_file_name(5,row.name) #mov.save(figure_path+figure_name+'.tif') mov.save(figure_path+figure_name+'_'+f'{time_cropping[0]}' + '_' + f'{time_cropping[1]}'+'.tif') return
mouse_row = selected_rows.iloc[0] mouse_row_new = main_source_extraction(mouse_row, parameters_source_extraction, dview) mouse_row_new = db.set_version_analysis('source_extraction', mouse_row_new) states_df = db.append_to_or_merge_with_states_df(states_df, mouse_row_new) db.save_analysis_states_database(states_df, path=analysis_states_database_path, backup_path = backup_path) #%% COMPONENT EVALUATION min_SNR = 3 # adaptive way to set threshold on the transient size r_values_min = 0.85 # threshold on space consistency (if you lower more components # will be accepted, potentially with worst quality) parameters_component_evaluation = {'min_SNR': min_SNR, 'rval_thr': r_values_min, 'use_cnn': False} cm.stop_server(dview=dview) mouse_row_new = main_component_evaluation(mouse_row_new, parameters_component_evaluation) component_evaluation_output = eval(mouse_row_new['component_evaluation_output']) input_hdf5_file_path = component_evaluation_output['main'] cnm = load_CNMF(input_hdf5_file_path) print('Accepted components = ') print(len(cnm.estimates.idx_components)) print('Rejected components = ') print(len(cnm.estimates.idx_components_bad))
def signal_to_noise(data_path, agonia_th, ground_truth=None, neurofinder=False): '''Calculate signal to noise ratio of each box Parameters ---------- data_path : string path to folder containing the data agonia_th : float threshold for detection confidence for each box''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load Caiman results cnm = cnmf.load_CNMF(results_caiman_path) # calculate the centers of the CaImAn factors centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() # keep only cells above confidence threshold boxes = boxes[boxes[:, 4] > agonia_th].astype('int') k = 0 for cell, box in enumerate(boxes): idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] if not idx_factor: boxes = np.delete(boxes, cell - k, axis=0) k += 1 if ground_truth is not None: stats = ag.compute_stats(np.array(ground_truth), boxes, iou_threshold=0.5) stnr = [] #np.zeros(boxes.shape[0]) if neurofinder: for cell, box in enumerate(boxes): if ground_truth is not None: if cell in stats['true_positives'][:, 1]: box_trace = images[:, box[1]:box[3], box[0]:box[2]] box_trace_arr = np.array(images[:, box[1]:box[3], box[0]:box[2]]) box_trace_arr_cut = box_trace_arr.copy() box_trace_arr_cut[box_trace < 30] = np.nan mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2)) trace_flat = box_trace[box_trace > 30] noise_box = np.std( trace_flat[trace_flat < np.percentile(trace_flat, 25)]) amp_trace = max(mean_box) - min(mean_box) stnr.append(amp_trace / noise_box) else: box_trace = images[:, box[1]:box[3], box[0]:box[2]] box_trace_arr = np.array(images[:, box[1]:box[3], box[0]:box[2]]) box_trace_arr_cut = box_trace_arr.copy() box_trace_arr_cut[box_trace < 30] = np.nan mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2)) trace_flat = box_trace[box_trace > 30] noise_box = np.std( trace_flat[trace_flat < np.percentile(trace_flat, 25)]) amp_trace = max(mean_box) - min(mean_box) stnr.append(amp_trace / noise_box) else: for cell, box in enumerate(boxes): if ground_truth is not None: if cell in stats['true_positives'][:, 1]: box_trace = images[:, box[1]:box[3], box[0]:box[2]] mean_box = box_trace.mean(axis=(1, 2)) amp_trace = max(mean_box) - min(mean_box) noise_box = np.std(box_trace[box_trace < np.percentile( box_trace[box_trace != 0], 25)]) stnr.append(amp_trace / noise_box) else: box_trace = images[:, box[1]:box[3], box[0]:box[2]] mean_box = box_trace.mean(axis=(1, 2)) amp_trace = max(mean_box) - min(mean_box) noise_box = np.std(box_trace[ box_trace < np.percentile(box_trace[box_trace != 0], 25)]) stnr.append(amp_trace / noise_box) return np.array(stnr)
def trace_correlation(data_path, agonia_th, select_cells=False, plot_results=True, denoise=False): '''Calculate the correlation between the mean of the Agonia Box and the CaImAn factor. Parameters ---------- data_path : string path to folder containing the data agonia_th : float threshold for detection confidence for each box select_cells: bool, optional if True get index of active cells plot_results: bool, optional if True do boxplot of correlation values for all cells, if selected_cells, use only active cells denoise : bool, optional if True subtract neuropil Returns ------- cell_corr : numpy array corrcoef value for all cells idx_active : list if select_cells=True returns index of active cells, otherwise returns index of all cells boxes_traces : NxT ndarray temporal trace for the N box that has an asociated caiman factor''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) # load Caiman results cnm = cnmf.load_CNMF(results_caiman_path) # calculate the centers of the CaImAn factors centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) # load boxes with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() # keep only cells above confidence threshold boxes = boxes[boxes[:, 4] > agonia_th].astype('int') #delete boxes that do not have a caiman cell inside k = 0 for cell, box in enumerate(boxes): idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] if not idx_factor: boxes = np.delete(boxes, cell - k, axis=0) k += 1 # Load video as 3D tensor (each plane is a frame) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') boxes_traces = np.empty((boxes.shape[0], images.shape[0])) #calculate correlations between boxes and CaImAn factors cell_corr = np.empty(len(boxes_traces)) neuropil_trace = np.zeros(T) if denoise: _, neuropil_trace, neuropil_power = substract_neuropil( data_path, agonia_th, 100, 80) for cell, box in enumerate(boxes): # calculate boxes traces as means over images #boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2))-neuropil_trace boxes_traces[cell] = images[:, box[1]:box[3], box[0]:box[2]].mean(axis=(1, 2)) #for using the percentile criteria med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0) box_trace = images[:, box[1]:box[3], box[0]:box[2]] boxes_traces[cell] = box_trace[:, np.logical_and( med > np.percentile(med, 80), med < np.percentile(med, 95))].mean( axis=1) boxes_traces[ cell] = boxes_traces[cell] - neuropil_trace * neuropil_power * .7 #boxes_traces[cell] = boxes_traces[cell]-neuropil_trace*boxes_traces[cell].mean() # get the asociated CaImAn factor by checking if its center of mass is inside the box idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] # in case there is more than one center inside the box choose the one closer to the center of the box if len(idx_factor) > 1: idx_factor = [ idx_factor[np.argmin([ np.linalg.norm([(box[3] - box[1]) / 2, (box[2] - box[0]) / 2] - c) for c in centers[idx_factor] ])] ] cell_corr[cell] = np.corrcoef( [cnm.estimates.C[idx_factor[0]], boxes_traces[cell]])[1, 0] if select_cells: #select only active cells using CaImAn criteria fitness, _, _, _ = compute_event_exceptionality(boxes_traces) idx_active = [cell for cell, fit in enumerate(fitness) if fit < -20] else: idx_active = [cell for cell, _ in enumerate(boxes_traces)] if plot_results: corr_toplot = [ corr for id, corr in enumerate(cell_corr) if id in idx_active and ~np.isnan(corr) ] fig, ax = plt.subplots() p = ax.boxplot(corr_toplot) ax.set_ylim([-1, 1]) plt.text(1.2, 0.8, 'n_cells = {}'.format(len(corr_toplot))) return cell_corr, idx_active, boxes_traces
'block_size_spat': block_size, 'num_blocks_per_run_spat': num_blocks_per_run, 'num_blocks_per_run_temp': num_blocks_per_run, 'n_pixels_per_process': n_pixels_per_process, 'ssub': 2, 'tsub': 2, 'p_tsub': 1, 'p_ssub': 1, 'thr_method': 'nrg' } init_method = global_params['init_method'] opts = params.CNMFParams(params_dict=params_dict) cnm = load_CNMF(fname_new[:-5] + '_cnmf_gsig.hdf5') # %% prepare ground truth masks gt_file = os.path.join( os.path.split(fname_new)[0], os.path.split(fname_new)[1][:-4] + 'match_masks.npz') with np.load(gt_file, encoding='latin1') as ld: print(ld.keys()) Cn_orig = ld['Cn'] gt_estimate = Estimates(A=scipy.sparse.csc_matrix(ld['A_gt'][()]), b=ld['b_gt'], C=ld['C_gt'], f=ld['f_gt'], R=ld['YrA_gt'], dims=(ld['d1'], ld['d2']))
max_ = img.max() else: min_, max_ = min_max img = (img - min_) / (max_ - min_) * gain img = img.astype(out_type) img = np.dstack([img] * 3) return img F = FileDialog() # load object saved by CNMF fpath = F.getOpenFileName(caption='Load CNMF Object', filter='HDF5 (*.h5 *.hdf5);;NWB (*.nwb)')[0] cnm_obj = load_CNMF(fpath) # movie if not os.path.exists(cnm_obj.mmap_file): M = FileDialog() cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file', filter='*.mmap')[0] if fpath[-3:] == 'nwb': mov = cm.load(cnm_obj.mmap_file, var_name_hdf5='acquisition/TwoPhotonSeries') else: mov = cm.load(cnm_obj.mmap_file) estimates = cnm_obj.estimates params_obj = cnm_obj.params
def substract_neuropil(data_path, agonia_th, neuropil_pctl, signal_pctl): '''Calculate the neuropil trace as the average fluorescence of every pixel that is not in a box. subtract it to every box trace, calculated as the activity of the pixels above the signal_pctl. Parameters ---------- data_path : string path to folder containing the data agonia_th : float threshold for detection confidence for each box neuropil_pctl : int percentile of pixels intensity bellow which pixel is considered to calculate neuropil signal_pctl : int percentile of pixels intensity above which pixel is considered to calculate trace Returns ------- denoised_traces : NxT ndarray temporal traces after neuropil subtraction for the N boxes neuropil_trace : array normalized temporal trace of background noise, length T neuropil_power : float mean of neuropil trace before normalization ''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load Caiman results cnm = cnmf.load_CNMF(results_caiman_path) # calculate the centers of the CaImAn factors centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() # keep only cells above confidence threshold boxes = boxes[boxes[:, 4] > agonia_th].astype('int') #delete boxes that do not have a caiman cell inside k = 0 for cell, box in enumerate(boxes): idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] if not idx_factor: boxes = np.delete(boxes, cell - k, axis=0) k += 1 boxes_traces = np.empty((boxes.shape[0], images.shape[0])) mask = np.zeros(images.shape[1:], dtype=bool) for cell, box in enumerate(boxes): #boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2)) med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0) box_trace = images[:, box[1]:box[3], box[0]:box[2]] boxes_traces[cell] = box_trace[:, med > np.percentile(med, signal_pctl )].mean(axis=1) mask[box[1].astype('int'):box[3].astype('int'), box[0].astype('int'):box[2].astype('int')] = 1 mask = (1 - mask).astype('bool') not_cell = images[:, mask] #not_cell_med = np.median(not_cell,axis=0) #neuropil_trace = not_cell[:,not_cell_med<np.percentile(not_cell_med,neuropil_pctl)].mean(axis=1) #denoised_traces = boxes_traces-neuropil_trace neuropil_trace = not_cell.mean(axis=1) neuropil_power = neuropil_trace.mean() neuropil_trace = neuropil_trace / neuropil_power denoised_traces = np.array([ boxes_traces[i] - neuropil_trace * boxes_traces[i].mean() for i in range(boxes_traces.shape[0]) ]) return denoised_traces, neuropil_trace, neuropil_power