def seeded_Caiman_wAgonia(data_path, opts, agonia_th): '''Run Caiman using as seeds the boxes detected with Agonia and save results Parameters ---------- data_path : string path to folder containing the data opts : caiman object with the options for motion correction agonia_th : float threshold for detection confidence for each box''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') #load frames in python format (T x X x Y) #%% restart cluster to clean up memory if 'dview' in locals(): cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=12, single_thread=False) # Load the boxes in the pickle file into the ROIs arrays. with open(boxes_path, 'rb') as f: cajas = pickle.load(f) f.close() cajas = cajas[cajas[:, 4] > agonia_th] #use the average size of the Agonia boxes as the cell size parameter for Caiman half_width = np.mean(cajas[:, 2] - cajas[:, 0]) / 2 half_height = np.mean(cajas[:, 3] - cajas[:, 1]) / 2 gSig = (half_width.astype(int), half_height.astype(int)) opts_dict = {'gSig': gSig} opts.change_params(opts_dict) #Build masks with the Agonia boxes Ain = np.zeros((np.prod(images.shape[1:]), cajas.shape[0]), dtype=bool) for i, box in enumerate(cajas): frame = np.zeros(images.shape[1:]) #note that the coordinates of the boxes are transpose with respect to Caiman objects frame[box[1].astype('int'):box[3].astype('int'), box[0].astype('int'):box[2].astype('int')] = 1 Ain[:, i] = frame.flatten('F') #Run analysis cnm_seeded = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain) try: print('initiating fit') cnm_seeded.fit(images) print('saving results') #cnm_seeded.estimates.detrend_df_f(quantileMin=50,frames_window=750,detrend_only=False) cnm_seeded.save( os.path.join( data_path, os.path.splitext(data_name[0])[0] + '_analysis_results.hdf5')) except: print('Tenemos un problema...') cm.stop_server(dview=dview)
def run_cnmf(n_processes, opts, dview, images): """The FOV is split is different overlapping patches that are subsequently processed in parallel by the CNMF algorithm. The results from all the patches are merged with special attention to idendtified components on the border. The results are then refined by additional CNMF iterations.""" # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0). If you want to have # deconvolution within each patch change params.patch['p_patch'] to a # nonzero value cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) return cnm
def run_pipeline(n_processes, opts, dview, do_mc=True, time_it=False): """Run the combined steps of motion correction, memory mapping, and cnmf fitting in one step.""" # Start timer start = time.time() # Run pipeline cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm1.fit_file(motion_correct=do_mc) # Print time if requested if time_it: print('Pipeline processing took {:.3f} seconds'.format(time.time() - start)) return cnm1
def do_fit(self): """ Perform the CNMF calculation. """ t = tic() print('Starting motion correction and CNMF...') cnm_seeded = cnmf.CNMF(self.n_processes, params=self.opts, dview=self.dview, Ain=self.Ain) cnm_seeded.fit(self.movie) self.coords = extract_cell_locs(cnm_seeded) cnm_seeded.estimates.detrend_df_f() self.dff = cnm_seeded.estimates.F_dff cnm_seeded.save(self.save_folder + f'caiman_data_{self.fnumber:04}.hdf5') print(f'CNMF fitting done. Took {toc(t):.4f}s') return cnm_seeded.estimates.C
def cnmf_neurofinder(params_dict): import numpy as np from caiman.source_extraction.cnmf import cnmf from caiman.source_extraction.cnmf import params fname_new = params_dict['fnames'][0] print(fname_new) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') opts = params.CNMFParams(params_dict=params_dict) dview = params_dict['dview'] print('Starting CNMF') opts.set('temporal', {'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) cnm.params.change_params({'update_background_components': True, 'skip_refinement': False, 'n_pixels_per_process': 4000, 'dview': dview}) opts.set('temporal', {'p': params_dict['p']}) cnm2 = cnm.refit(images, dview=dview) cnm2.save(fname_new[:-5] + '_cnmf.hdf5') return cnm2
def do_final_fit(self): """ Do the last fit on all the tiffs in the folder. This makes an entirely concenated cnmf fit. """ t = tic() print(f'processing files: {self.tiffs}') self.opts.change_params(dict(fnames=self.tiffs)) maplist = [] for i in range(self.fnumber): m = glob(self.folder + f'MAP{i}a_*')[0] maplist.append(m) # all_memmaps = glob(self.folder + 'MAP00*.mmap') memmap = cm.save_memmap_join(maplist, base_name='FINAL', dview=self.dview) Yr, dims, T = cm.load_memmap(memmap) images = np.reshape(Yr.T, [T] + list(dims), order='F') cnm_seeded = cnmf.CNMF(self.n_processes, params=self.opts, dview=self.dview, Ain=self.Ain) cnm_seeded.fit(images) cnm_seeded.save(self.save_folder + 'FINAL_caiman_data.hdf5') self.coords = cm.utils.visualization.get_contours( cnm_seeded.estimates.A, dims=cnm_seeded.dims) cnm_seeded.estimates.detrend_df_f() self.dff = cnm_seeded.estimates.F_dff self.C = cnm_seeded.estimates.C #self.save_json() print(f'CNMF fitting done. Took {toc(t):.4f}s') print('Caiman online analysis done.')
def compute_cnmf(n_processes, dview, images, dims): '''Applying CNMF algorithm on the video Args: n_processes: Number of parallel processing dview: Direct View object for parallelization pruposes when using ipyparallel images: The input video shape: (frames, dims, dims) dims: Dimension of each frame shape: (521, 521) Returns: cnm: Object obtained from CNMF algorithm with C,A,S,b,f ''' cnm = cnmf.CNMF(n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview, gnb=gnb, rf=rf, stride=stride, rolling_sum=False) cnm = cnm.fit(images) return cnm
# expected half size of neurons gSig = params_movie['gSig'] # this controls sparsity alpha_snmf = params_movie['alpha_snmf'] # frame rate of movie (even considering eventual downsampling) final_frate = params_movie['final_frate'] if params_movie['is_dendrites'] == True: if params_movie['init_method'] is not 'sparse_nmf': raise Exception('dendritic requires sparse_nmf') if params_movie['alpha_snmf'] is None: raise Exception('need to set a value for alpha_snmf') #%% Extract spatial and temporal components on patches t1 = time.time() cnm = cnmf.CNMF(n_processes, k=K, gSig=gSig, merge_thresh=0.8, p=0, dview=dview, Ain=None, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=True, gnb=1, method_deconvolution='oasis', remove_very_bad_comps=True) cnm = cnm.fit(images) A_tot = cnm.A C_tot = cnm.C YrA_tot = cnm.YrA b_tot = cnm.b f_tot = cnm.f sn_tot = cnm.sn t2 = time.time() - t1 print(('Number of components:' + str(A_tot.shape[-1]))) #%% np.savez(os.path.join(os.path.split(fname_new)[0], os.path.split(fname_new)[ 1][:-4] + 'results_analysis_ZEBRA_patch.npz'), Cn=Cn, A=A_tot, C=C_tot, b=b_tot, f=f_tot, YrA=YrA_tot, sn=sn_tot, d1=d1, d2=d2) #%% pl.figure()
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie, m_els.resize(1, 1, ds_ratio)], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = {'fnames': fnames, 'p': p, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub} opts.change_params(params_dict=opts_dict); # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0). If you want to have # deconvolution within each patch change params.patch['p_patch'] to a # nonzero value #opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cns = local_correlations_movie_offline(mc.mmap_file[0], remove_baseline=True, window=1000, stride=1000, winSize_baseline=100, quantil_min_baseline=10, dview=dview) Cn = Cns.max(axis=0) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% save results cnm.estimates.Cn = Cn cnm.save(fname_new[:-5]+'_init.hdf5') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% cnm2.estimates.Cn = Cn cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
def main(): pass # For compatibility between running under Spyder and the CLI #%% """ General parameters """ play_movie = 1 plot_extras = 1 plot_extras_cell = 1 compute_mc_metrics = 1 #%% Select file(s) to be processed (download if not present) """ Load file """ #fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed #if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: # fnames = [download_demo(fnames[0])] #fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif'] #f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\' f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\' f_name = 'Ch1' f_ext = 'tif' fnames = [f_dir + f_name + '.' + f_ext] #fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5'] #%% First setup some parameters for data and motion correction """ Parameters """ # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 1 #0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels #max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] max_shifts = [6, 6] # start a new patch for pw-rigid motion correction every x pixels #strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) strides = [48, 48] # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q if play_movie: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # type "mc."and press TAB to see all interesting associated variables and self. outputs # interesting outputs # saved file is mc.fname_tot_els / mc.fname_tot_rig # mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch # mc.coord_shifts_els: coordinates associated to patches with shifts # mc.total_template_els: updated template for pw # mc.total_template_rig: updated template for rigid # mc.templates_rig: templates for each iteration in rig #%% # compute metrics for the results (TAKES TIME!!) if compute_mc_metrics: # not finished bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) final_size = np.subtract( mc.total_template_els.shape, 2 * bord_px_els) # remove pixels in the boundaries winsize = 100 swap_dim = False resize_fact_flow = .2 # downsample for computing ROF tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction( fnames[0], final_size[0], final_size[1], swap_dim, winsize=winsize, play_flow=False, resize_fact_flow=resize_fact_flow) plt.figure() plt.plot(correlations_orig) # %% compare with original movie if play_movie: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit del m_orig del m_els if plot_extras: # plot total template plt.figure() plt.imshow(mc.total_template_els) plt.title('Template after iteration') # plot x and y corrections plt.figure() plt.plot(mc.shifts_rig) plt.title('Rigid motion correction xy movement') plt.legend(['x shift', 'y shift']) plt.xlabel('frames') # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 2 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 2 # number of components per patch gSig = [15, 15] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 1 # spatial subsampling during initialization tsub = 1 # temporal subsampling during intialization # parameters for component evaluation opts_dict = { 'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub } opts.change_params(params_dict=opts_dict) # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) if plot_extras_cell: num_cell_plot = 51 plt.figure() plt.plot(cnm.estimates.C[num_cell_plot, :]) plt.title('Temporal component') plt.legend(['Cell ' + str(num_cell_plot)]) # plot component sptial profile A # first convert back to dense components plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape( list(dims)) plt.figure() plt.imshow(plot_spat_A) plt.title('Spatial component cell ' + str(num_cell_plot)) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') if plot_extras: plt.figure() plt.imshow(Cn) plt.title('Local correlations') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.90 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Component selection: min_SNR=' + str(min_SNR) + '; rval_thr=' + str(rval_thr) + '; cnn prob range=[' + str(cnn_lowest) + ' ' + str(cnn_thr) + ']') # %% VIEW TRACES (accepted and rejected) if plot_extras: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Accepted') cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) plt.suptitle('Rejected') #plt.figure(); #plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:]) # # # # #plt.figure(); #plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]); #plt.plot(); #plt.show(); # # #plt.figure(); #plt.plot(cnm2.estimates.detrend_df_f[1,:]) # these store the good and bad components, and next step sorts them # cnm2.estimates.idx_components # cnm2.estimates.idx_components_bad #%% update object with selected components #cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) plt.suptitle("Final results") #%% Save the mc data as in cmn struct as well ## #mc_out = dict( # pw_rigid = mc.pw_rigid, # fname = mc.fname, # mmap_file = mc.mmap_file, # total_template_els = mc.total_template_els, # total_template_rig = mc.total_template_rig, # border_nan = mc.border_nan, # border_to_0 = mc.border_to_0, # x_shifts_els = mc.x_shifts_els, # y_shifts_els = mc.y_shifts_els, # Cn = Cn # ) # # #deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out) #%% reconstruct denoised movie (press q to exit) if play_movie: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) save_results = True if save_results: cnm2.save(fnames[0][:-4] + '_results.hdf5')
raise Exception('need to set a value for alpha_snmf') #%% t1 = time.time() n_pixels_per_process = 4000 # the smaller the best memory performance block_size = 50000 cnm = cnmf.CNMF(n_processes, k=K, gSig=gSig, merge_thresh=0.8, p=0, dview=dview, Ain=None, rf=rf, stride=stride, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=True, gnb=1, method_deconvolution='oasis', n_pixels_per_process=n_pixels_per_process, p_ssub=2, p_tsub=2, block_size=block_size, check_nan=False) cnm = cnm.fit(images) A_tot = cnm.A C_tot = cnm.C YrA_tot = cnm.YrA b_tot = cnm.b
K = 25 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 1 # order of the autoregressive system gnb = 1 # global background order border_pix = 5 #%% Now RUN CNMF cnm = cnmf.CNMF(n_processes, method_init='sparse_nmf', k=K, gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview, gnb=gnb, rf=rf, stride=stride, rolling_sum=False, alpha_snmf=0.1, border_pix=border_pix, only_init_patch=False) cnm = cnm.fit(images) #%% plot contour plots of components plt.figure() crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of components') #%% cm.movie( np.reshape(Yr - cnm.A.dot(cnm.C) - cnm.b.dot(cnm.f),
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set this to true to show movies and plots fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. To # close the video press q m_orig = cm.load_movie_chain(fname[:1]) downsample_ratio = 0.2 offset_mov = -np.min(m_orig[:100]) moviehandle = m_orig.resize(1, 1, downsample_ratio) if display_images: moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # maximum shift to be used for trimming against NaNs #%% compare with original movie moviehandle = cm.concatenate([ m_orig.resize(1, 1, downsample_ratio) + offset_mov, m_els.resize(1, 1, downsample_ratio) ], axis=2) display_images = False if display_images: moviehandle.play(fr=60, q_max=99.5, magnification=2, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) #%% PLOT COMPONENTS if display_images: plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) if display_images: view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=Cn) view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components_bad], cnm.C[idx_components_bad], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components_bad], img=Cn) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape( dims + (-1, ), order='F').transpose([2, 0, 1]) #%% play along side original data moviehandle = cm.concatenate([ m_els.resize(1, 1, downsample_ratio), denoised.resize(1, 1, downsample_ratio) ], axis=2) if display_images: moviehandle.play(fr=60, gain=15, magnification=2, offset=0) # press q to exit
alpha_snmf = params_movie['alpha_snmf'] # frame rate of movie (even considering eventual downsampling) final_frate = params_movie['final_frate'] if params_movie['is_dendrites'] == True: if params_movie['init_method'] is not 'sparse_nmf': raise Exception('dendritic requires sparse_nmf') if params_movie['alpha_snmf'] is None: raise Exception('need to set a value for alpha_snmf') # %% Extract spatial and temporal components on patches t1 = time.time() # TODO: todocument # TODO: warnings 3 cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=params_movie[ 'only_init_patch'], gnb=params_movie['gnb'], method_deconvolution='oasis') comp.cnmpatch = copy.copy(cnm) cnm = cnm.fit(images) A_tot = cnm.A C_tot = cnm.C YrA_tot = cnm.YrA b_tot = cnm.b f_tot = cnm.f sn_tot = cnm.sn comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1 comp.comparison['cnmf_on_patch']['ourdata'] = [cnm.A.copy(), cnm.C.copy()] print(('Number of components:' + str(A_tot.shape[-1]))) # %%
def run(): date_recorded = '190503' mouse_id = 'M439939' play_movie = False resolution = 512 channel = 'green' data_folder = r"\\allen\programs\braintv\workgroups\nc-ophys\Jun\raw_data\{}-{}-2p" \ r"\110_LSNDGC_reorged".format(date_recorded, mouse_id) curr_folder = os.path.dirname(os.path.realpath(__file__)) plane_ns = [ f for f in os.listdir(data_folder) if os.path.isdir(f) and f[:5] == 'plane' ] plane_ns.sort() print('planes:') print('\n'.join(plane_ns)) # %% start cluster c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=6, single_thread=False) for plane_n in plane_ns: print('\nsegmenting plane: {}'.format(plane_n)) plane_folder = os.path.join(data_folder, plane_n, channel, 'corrected') os.chdir(plane_folder) fn = [f for f in os.listdir(plane_folder) if f[-5:] == '.mmap'] if len(fn) > 1: print('\n'.join(fn)) raise LookupError('more than one file found.') elif len(fn) == 0: raise LookupError('no file found.') else: fn = fn[0] fn_parts = fn.split('_') d1 = int(fn_parts[fn_parts.index('d1') + 1]) # column, x d2 = int(fn_parts[fn_parts.index('d2') + 1]) # row, y d3 = int(fn_parts[fn_parts.index('d3') + 1]) # channel d4 = int(fn_parts[fn_parts.index('frames') + 1]) # frame, T order = fn_parts[fn_parts.index('order') + 1] print('playing {} ...'.format(fn)) mov = np.memmap(filename=fn, shape=(d1, d2, d4), order=order, dtype=np.float32, mode='r') mov = mov.transpose((2, 1, 0)) print('shape of joined movie: {}.'.format(mov.shape)) #%% play movie, press q to quit if play_movie: cm.movie(mov).play(fr=50, magnification=1, gain=2.) #%% movie cannot be negative! mov_min = float(np.amin(mov)) print('minimum pixel value: {}.'.format(mov_min)) if mov_min < 0: raise Exception( 'Movie too negative, add_to_movie should be larger') #%% correlation image. From here infer neuron size and density Cn = cm.movie(mov).local_correlations(swap_dim=False) # plt.imshow(Cn, cmap='gray') # plt.show() K = 100 # number of neurons expected per patch gSig = [5, 5] # expected half size of neurons merge_thresh = 0.9 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system cnm = cnmf.CNMF( n_processes, k=10, # number of neurons expected per patch gSig=[5, 5], # expected half size of neurons merge_thresh=0.9, # merging threshold, max correlation allowed p=2, # order of the autoregressive system dview=dview, Ain=None, method_deconvolution='oasis', rolling_sum=False, method_init='sparse_nmf', alpha_snmf=10e1, ssub=1, tsub=1, p_ssub=1, p_tsub=1, rf=int(resolution / 2), # half-size of the patches in pixels border_pix=0, do_merge=False) cnm = cnm.fit(mov) A, C, b, f, YrA, sn = cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, cnm.sn #%% # crd = cm.utils.visualization.plot_contours(cnm.A, Cn) # plt.show() # input("Press enter to continue ...") roi_num = cnm.A.shape[1] save_fn = h5py.File('caiman_segmentation_results.hdf5') bias = save_fn['bias_added_to_movie'].value save_fn['masks'] = np.array(cnm.A.todense()).T.reshape( (roi_num, 512, 512), order='F') save_fn['traces'] = cnm.C - bias save_fn.close() copyfile( os.path.join(plane_folder, 'caiman_segmentation_results.hdf5'), os.path.join(curr_folder, plane_n, 'caiman_segmentation_results.hdf5')) plt.close('all')
def perform_cnmf(video, params, roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints, use_multiprocessing=True): if use_multiprocessing: backend = 'multiprocessing' cm.stop_server() c, dview, n_processes = cm.cluster.setup_cluster(backend=backend, n_processes=None, single_thread=False) else: dview = None # print("~") # print(roi_spatial_footprints.shape) video_path = "video_temp.tif" tifffile.imsave(video_path, video) # dataset dependent parameters fnames = [video_path] # filename to be processed fr = params['imaging_fps'] # imaging rate in frames per second decay_time = params['decay_time'] # length of a typical transient in seconds # parameters for source extraction and deconvolution p = params['autoregressive_order'] # order of the autoregressive system gnb = params['num_bg_components'] # number of global background components merge_thresh = params['merge_threshold'] # merging threshold, max correlation allowed rf = None # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = roi_spatial_footprints.shape[1] # number of components per patch gSig = [params['half_size'], params['half_size']] # expected half size of neurons init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') rolling_sum = True rolling_length = 50 is_dendrites = False # flag for analyzing dendritic data alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF # parameters for component evaluation min_SNR = params['min_snr'] # signal to noise ratio for accepting a component rval_thr = params['min_spatial_corr'] # space correlation threshold for accepting a component cnn_thr = params['cnn_threshold'] # threshold for CNN based classifier border_pix = 0 fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C') # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') cnm = cnmf.CNMF(n_processes=8, k=K, gSig=gSig, merge_thresh= merge_thresh, p = p, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, rolling_sum=rolling_sum, only_init_patch = True, skip_refinement=True, gnb = gnb, border_pix = border_pix, ssub=1, ssub_B=1, tsub=1, Ain=roi_spatial_footprints, Cin=roi_temporal_footprints, b_in=bg_spatial_footprints, f_in=bg_temporal_footprints, do_merge=False) cnm = cnm.fit(images) roi_spatial_footprints = cnm.A roi_temporal_footprints = cnm.C roi_temporal_residuals = cnm.YrA bg_spatial_footprints = cnm.b bg_temporal_footprints = cnm.f if use_multiprocessing: if backend == 'multiprocessing': dview.close() else: try: dview.terminate() except: dview.shutdown() cm.stop_server() return roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints
raise Exception('dendritic requires sparse_nmf') if params_movie['alpha_snmf'] is None: raise Exception('need to set a value for alpha_snmf') # %% Extract spatial and temporal components on patches t1 = time.time() border_pix = 0 # if motion correction introduces problems at the border remove pixels from border # TODO: todocument # TODO: warnings 3 cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=params_movie['only_init_patch'], gnb=params_movie['gnb'], method_deconvolution='oasis', border_pix=border_pix, low_rank_background=params_movie['low_rank_background']) cnm = cnm.fit(images) A_tot = cnm.A C_tot = cnm.C YrA_tot = cnm.YrA b_tot = cnm.b f_tot = cnm.f sn_tot = cnm.sn
# %% Extract spatial and temporal components # TODO: todocument t1 = time.time() if images.shape[0] > 10000: check_nan = False else: check_nan = True cnm = cnmf.CNMF(check_nan=check_nan, n_processes=1, k=A_in.shape[-1], gSig=[radius, radius], merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], Ain=A_in.astype(np.bool), dview=dview, rf=None, stride=None, gnb=params_movie['gnb'], method_deconvolution='oasis', border_pix=0, low_rank_background=params_movie['low_rank_background'], n_pixels_per_process=1000) cnm = cnm.fit(images) A = cnm.A C = cnm.C YrA = cnm.YrA b = cnm.b f = cnm.f sn = cnm.sn
def main(): pass # For compatibility between running under Spyder and the CLI #%% start a cluster c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% save files to be processed # This datafile is distributed with Caiman fnames = [ os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif') ] # location of dataset (can actually be a list of filed to be concatenated) add_to_movie = -np.min(cm.load(fnames[0], subindices=range(200))).astype(float) # determine minimum value on a small chunk of data add_to_movie = np.maximum(add_to_movie, 0) # if minimum is negative subtract to make the data non-negative base_name = 'Yr' fname_new = cm.save_memmap(fnames, dview=dview, base_name=base_name, order='C', add_to_movie=add_to_movie) #%% LOAD MEMORY MAPPABLE FILE Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') #%% play movie, press q to quit play_movie = False if play_movie: cm.movie(images[1400:]).play(fr=50, magnification=4, gain=3.) #%% correlation image. From here infer neuron size and density Cn = cm.movie(images).local_correlations(swap_dim=False) plt.imshow(Cn, cmap='gray') plt.title('Correlation Image') #%% set up some parameters is_patches = True # flag for processing in patches or not if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 10 # half size of each patch stride = 4 # overlap between patches K = 4 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = 30 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order #%% Now RUN CNMF cnm = cnmf.CNMF(n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview, gnb=gnb, rf=rf, stride=stride, rolling_sum=False) cnm = cnm.fit(images) #%% plot contour plots of components plt.figure() crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of components') #%% A_in, C_in, b_in, f_in = cnm.A[:, :], cnm.C[:], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) fr = 10 # approximate frame rate of data decay_time = 5.0 # length of transient min_SNR = 2.5 # peak SNR for accepted components (if above this, acept) rval_thr = 0.90 # space correlation threshold (if above this, accept) use_cnn = True # use the CNN classifier min_cnn_thr = 0.95 # if cnn classifier predicts below this value, reject idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=use_cnn, thresh_cnn_min=min_cnn_thr) #%% visualize selected and rejected components plt.figure() plt.subplot(1, 2, 1) cm.utils.visualization.plot_contours(cnm2.A[:, idx_components], Cn, thr=0.9) plt.title('Selected components') plt.subplot(1, 2, 2) plt.title('Discaded components') cm.utils.visualization.plot_contours(cnm2.A[:, idx_components_bad], Cn, thr=0.9) #%% plt.figure() crd = cm.utils.visualization.plot_contours(cnm2.A.tocsc()[:, idx_components], Cn, thr=0.9) plt.title('Contour plots of components') #%% visualize selected components cm.utils.visualization.view_patches_bar(Yr, cnm2.A.tocsc()[:, idx_components], cnm2.C[idx_components, :], cnm2.b, cnm2.f, dims[0], dims[1], YrA=cnm2.YrA[idx_components, :], img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server() log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file)
def main(): pass # For compatibility between running under Spyder and the CLI # %% start a cluster c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% set up some parameters fnames = [ os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif') ] # file(s) to be analyzed is_patches = True # flag for processing in patches or not fr = 10 # approximate frame rate of data decay_time = 5.0 # length of transient if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 10 # half size of each patch stride = 4 # overlap between patches K = 4 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = 30 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order params_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'rf': rf, 'stride': stride, 'K': K, 'gSig': gSig, 'merge_thr': merge_thresh, 'p': p, 'nb': gnb } opts = params.CNMFParams(params_dict=params_dict) # %% Now RUN CaImAn Batch (CNMF) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit_file() # %% plot contour plots of components Cns = local_correlations_movie_offline(fnames[0], remove_baseline=True, swap_dim=False, window=1000, stride=1000, winSize_baseline=100, quantil_min_baseline=10, dview=dview) Cn = Cns.max(axis=0) cnm.estimates.plot_contours(img=Cn) # %% load memory mapped file Yr, dims, T = cm.load_memmap(cnm.mmap_file) images = np.reshape(Yr.T, [T] + list(dims), order='F') # %% refit cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) min_SNR = 2 # peak SNR for accepted components (if above this, acept) rval_thr = 0.85 # space correlation threshold (if above this, accept) use_cnn = True # use the CNN classifier min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': use_cnn, 'min_cnn_thr': min_cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% visualize selected and rejected components cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% visualize selected components cnm2.estimates.view_components(images, idx=cnm2.estimates.idx_components, img=Cn) #%% only select high quality components (destructive) # cnm2.estimates.select_components(use_object=True) # cnm2.estimates.plot_contours(img=Cn) #%% save results cnm2.estimates.Cn = Cn cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') # %% play movie with results (original, reconstructed, amplified residual) cnm2.estimates.play_movie(images, magnification=4) # %% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file)
# %% Extract spatial and temporal components on patches # TODO: todocument if images.shape[0] > 10000: check_nan = False else: check_nan = True cnm = cnmf.CNMF(check_nan=check_nan, n_processes=1, k=A_in.shape[-1], gSig=[radius, radius], merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], Ain=A_in.astype(np.bool), dview=dview, rf=None, stride=None, gnb=params_movie['gnb'], method_deconvolution='oasis', border_pix=0, low_rank_background=params_movie['low_rank_background'], n_pixels_per_process=1000) cnm = cnm.fit(images) A = cnm.A C = cnm.C YrA = cnm.YrA b = cnm.b f = cnm.f snt = cnm.sn
def cnmf_patches(args_in): import numpy as np import caiman as cm import time import logging from caiman.source_extraction.cnmf import cnmf as cnmf # file_name, idx_,shapes,p,gSig,K,fudge_fact=args_in file_name, idx_, shapes, options = args_in name_log = os.path.basename(file_name[:-5]) + '_LOG_ ' + str( idx_[0]) + '_' + str(idx_[-1]) logger = logging.getLogger(name_log) hdlr = logging.FileHandler('./' + name_log) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.INFO) p = options['temporal_params']['p'] logger.info('START') logger.info('Read file') Yr, _, _ = load_memmap(file_name) Yr = Yr[idx_, :] if (np.sum(np.abs(np.diff(Yr)))) > 0.1: #Yr.filename=file_name d, T = Yr.shape # Y=np.reshape(Yr,(shapes[1],shapes[0],T),order='F') # Y.filename=file_name # dims = shapes[1],shapes[0] dims = shapes #shapes[1],shapes[0],shapes[2] images = np.reshape(Yr.T, [T] + list(dims), order='F') #images.filename=file_name cnm = cnmf.CNMF(n_processes = 1, k = options['init_params']['K'], gSig = options['init_params']['gSig'], merge_thresh = options['merging']['thr'], p = p, dview = None, Ain = None, Cin = None, f_in = None, do_merge = True,\ ssub = options['init_params']['ssub'], tsub = options['init_params']['ssub'], p_ssub = 1, p_tsub = 1, method_init = options['init_params']['method'], alpha_snmf = options['init_params']['alpha_snmf'],\ rf=None,stride=None, memory_fact=1, gnb = options['init_params']['nb'],\ only_init_patch = options['patch_params']['only_init']\ ,method_deconvolution = options['temporal_params']['method'], n_pixels_per_process = options['preprocess_params']['n_pixels_per_process'],\ block_size = options['temporal_params']['block_size'], check_nan = options['preprocess_params']['check_nan'],\ skip_refinement = options['patch_params']['skip_refinement'],N_iterations_refinement=options['patch_params']['nIter']) cnm = cnm.fit(images) Yr = [] images = [] return idx_, shapes, scipy.sparse.coo_matrix( cnm.A ), cnm.b, cnm.C, cnm.f, cnm.S, cnm.bl, cnm.c1, cnm.neurons_sn, cnm.g, cnm.sn, cnm.options, cnm.YrA.T # [d1,d2,T]=Y.shape # # options['spatial_params']['dims']=(d1,d2) # logger.info('Preprocess Data') # Yr,sn,g,psx=cm.source_extraction.cnmf.pre_processing.preprocess_data(Yr,**options['preprocess_params']) # # # logger.info('Initialize Components') # # Ain, Cin, b_in, f_in, center=cm.source_extraction.cnmf.initialization.initialize_components(Y, **options['init_params']) # # nA = np.squeeze(np.array(np.sum(np.square(Ain),axis=0))) # # nr=nA.size # Cin=coo_matrix(Cin) # # # YA = (Ain.T.dot(Yr).T)*scipy.sparse.spdiags(old_div(1.,nA),0,nr,nr) # AA = ((Ain.T.dot(Ain))*scipy.sparse.spdiags(old_div(1.,nA),0,nr,nr)) # YrA = YA - Cin.T.dot(AA) # Cin=Cin.todense() # # if options['patch_params']['only_init']: # # return idx_,shapes, coo_matrix(Ain), b_in, Cin, f_in, None, None , None, None, g, sn, options, YrA.T # # else: # # raise Exception('Bug here, need to double check. For now set ["patch_params"]["only_init"] = True') # logger.info('Spatial Update') # A,b,Cin, f_in = cm.source_extraction.cnmf.spatial.update_spatial_components(Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params']) # options['temporal_params']['p'] = 0 # set this to zero for fast updating without deconvolution # # import pdb # pdb.set_trace() # # logger.info('Temporal Update') # C,f,S,bl,c1,neurons_sn,g,YrA = cm.source_extraction.cnmf.temporal.update_temporal_components(Yr,A,b,Cin,f_in,bl=None,c1=None,sn=None,g=None,**options['temporal_params']) # # logger.info('Merge Components') # A_m,C_m,nr_m,merged_ROIs,S_m,bl_m,c1_m,sn_m,g_m=cm.source_extraction.cnmf.merging.merge_components(Yr,A,b,C,f,S,sn,options['temporal_params'], options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g, thr=options['merging']['thr'], fast_merge = True) # # logger.info('Update Spatial II') # A2,b2,C2,f = cm.source_extraction.cnmf.spatial.update_spatial_components(Yr, C_m, f, A_m, sn=sn, **options['spatial_params']) # # logger.info('Update Temporal II') # options['temporal_params']['p'] = p # set it back to original value to perform full deconvolution # C2,f2,S2,bl2,c12,neurons_sn2,g21,YrA = cm.source_extraction.cnmf.temporal.update_temporal_components(Yr,A2,b2,C2,f,bl=None,c1=None,sn=None,g=None,**options['temporal_params']) # # # Y=[] # Yr=[] # # logger.info('Done!') # return idx_,shapes,A2,b2,C2,f2,S2,bl2,c12,neurons_sn2,g21,sn,options,YrA else: return None
raise Exception('Movie contains nan! You did not remove enough borders') #%% Cn = cm.local_correlations(Y[:, :, :3000]) pl.imshow(Cn, cmap='gray') #%% if not is_patches: K = 35 # number of neurons expected per patch gSig = [7, 7] # expected half size of neurons merge_thresh = 0.8 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system cnm = cnmf.CNMF(n_processes, method_init=init_method, k=K, gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview, Ain=None, method_deconvolution='oasis', skip_refinement=False) cnm = cnm.fit(images) crd = plot_contours(cnm.A, Cn, thr=0.9) C_dff = extract_DF_F(Yr, cnm.A, cnm.C, cnm.bl, quantileMin=8, frames_window=200, dview=dview) pl.figure() pl.plot(C_dff.T)
def demix_and_deconvolve_with_cnmf(scan, num_components=200, AR_order=2, merge_threshold=0.8, num_processes=20, num_pixels_per_process=5000, block_size=10000, num_background_components=4, init_method='greedy_roi', soma_radius=(5, 5), snmf_alpha=None, init_on_patches=False, patch_downsampling_factor=None, percentage_of_patch_overlap=None): """ Extract spike train activity from multi-photon scans using CNMF. Uses constrained non-negative matrix factorization to find neurons/components (locations) and their fluorescence traces (activity) in a timeseries of images, and deconvolves them using an autoregressive model of the calcium impulse response function. See Pnevmatikakis et al., 2016 for details. Default values work alright for somatic images. :param np.array scan: 3-dimensional scan (image_height, image_width, num_frames). :param int num_components: An estimate of neurons/spatial components in the scan. :param int AR_order: Order of the autoregressive process used to model the impulse response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay. :param int merge_threshold: Maximal temporal correlation allowed between activity of overlapping components before merging them. :param int num_processes: Number of processes to run in parallel. None for as many processes as available cores. :param int num_pixels_per_process: Number of pixels that a process handles each iteration. :param int block_size: 'number of pixels to process at the same time for dot product' :param int num_background_components: Number of background components to use. :param string init_method: Initialization method for the components. 'greedy_roi':Look for a gaussian-shaped patch, apply rank-1 NMF, store components, calculate residual scan and repeat for num_components. 'sparse_nmf': Regularized non-negative matrix factorization (as impl. in sklearn) 'local_nmf': ... :param (float, float) soma_radius: Estimated neuron radius (in pixels) in y and x. Used in'greedy_roi' initialization to define the size of the gaussian window. :param int snmf_alpha: Regularization parameter (alpha) for the sparse NMF (if used). :param bool init_on_patches: If True, run the initialization methods on small patches of the scan rather than on the whole image. :param int patch_downsampling_factor: Division to the image dimensions to obtain patch dimensions, e.g., if original size is 256 and factor is 10, patches will be 26x26 :param int percentage_of_patch_overlap: Patches are sampled in a sliding window. This controls how much overlap is between adjacent patches (0 for none, 0.9 for 90%) :returns Location matrix (image_height x image_width x num_components). Inferred location of each component. :returns Activity matrix (num_components x num_frames). Inferred fluorescence traces (spike train convolved with the fitted impulse response function). :returns: Inferred location matrix for background components (image_height x image_width x num_background_components). :returns: Inferred activity matrix for background components (image_height x image_width x num_background_components). :returns: Raw fluorescence traces (num_components x num_frames) obtained from the scan minus activity from background and other components. :returns: Spike matrix (num_components x num_frames). Deconvolved spike activity. :returns: Autoregressive process coefficients (num_components x AR_order) used to model the calcium impulse response of each component: c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ... ..note:: Based on code provided by Andrea Giovanucci. ..note:: The produced number of components is not exactly what you ask for because some components will be merged or deleted. ..warning:: Computation- and memory-intensive for big scans. """ import caiman from caiman.source_extraction.cnmf import cnmf # Save as memory mapped file in F order (that's how caiman wants it) mmap_filename = _save_as_memmap(scan, base_name='/tmp/caiman', order='F').filename # 'Load' scan mmap_scan, (image_height, image_width), num_frames = caiman.load_memmap(mmap_filename) images = np.reshape(mmap_scan.T, (num_frames, image_height, image_width), order='F') # Start the ipyparallel cluster client, direct_view, num_processes = caiman.cluster.setup_cluster( n_processes=num_processes) # Optionally, run the initialization method in small patches to initialize components initial_A = None initial_C = None initial_f = None if init_on_patches: # Calculate patch size (only square patches allowed) bigger_dimension = max(image_height, image_width) smaller_dimension = min(image_height, image_width) patch_size = bigger_dimension / patch_downsampling_factor patch_size = min(patch_size, smaller_dimension) # if bigger than small dimension # Calculate num_components_per_patch num_nonoverlapping_patches = (image_height/patch_size) * (image_width/patch_size) num_components_per_patch = num_components / num_nonoverlapping_patches num_components_per_patch = max(num_components_per_patch, 1) # at least 1 # Calculate patch overlap in pixels overlap_in_pixels = patch_size * percentage_of_patch_overlap # Make sure they are integers patch_size = int(round(patch_size)) num_components_per_patch = int(round(num_components_per_patch)) overlap_in_pixels = int(round(overlap_in_pixels)) # Run CNMF on patches (only for initialization, no impulse response modelling p=0) model = cnmf.CNMF(num_processes, only_init_patch=True, p=0, rf=int(round(patch_size / 2)), stride=overlap_in_pixels, k=num_components_per_patch, merge_thresh=merge_threshold, method_init=init_method, gSig=soma_radius, alpha_snmf=snmf_alpha, gnb=num_background_components, n_pixels_per_process=num_pixels_per_process, block_size=block_size, check_nan=False, dview=direct_view, method_deconvolution='cvxpy') model = model.fit(images) # Delete log files (one per patch) log_files = glob.glob('caiman*_LOG_*') for log_file in log_files: os.remove(log_file) # Get results initial_A = model.A initial_C = model.C initial_f = model.f # Run CNMF model = cnmf.CNMF(num_processes, k=num_components, p=AR_order, merge_thresh=merge_threshold, gnb=num_background_components, method_init=init_method, gSig=soma_radius, alpha_snmf=snmf_alpha, n_pixels_per_process=num_pixels_per_process, block_size=block_size, check_nan=False, dview=direct_view, Ain=initial_A, Cin=initial_C, f_in=initial_f, method_deconvolution='cvxpy') model = model.fit(images) # Get final results location_matrix = model.A # pixels x num_components activity_matrix = model.C # num_components x num_frames background_location_matrix = model.b # pixels x num_background_components background_activity_matrix = model.f # num_background_components x num_frames spikes = model.S # num_components x num_frames, spike_ traces raw_traces = model.C + model.YrA # num_components x num_frames AR_coefficients = model.g # AR_order x num_components # Reshape spatial matrices to be image_height x image_width x num_frames new_shape = (image_height, image_width, -1) location_matrix = location_matrix.toarray().reshape(new_shape, order='F') background_location_matrix = background_location_matrix.reshape(new_shape, order='F') AR_coefficients = np.array(list(AR_coefficients)) # unwrapping it (num_components x 2) # Stop ipyparallel cluster client.close() caiman.stop_server() # Delete memory mapped scan os.remove(mmap_filename) return (location_matrix, activity_matrix, background_location_matrix, background_activity_matrix, raw_traces, spikes, AR_coefficients)
'ssub': global_params['ssub'], 'tsub': global_params['tsub'], 'thr_method': 'nrg' } init_method = global_params['init_method'] opts = params.CNMFParams(params_dict=params_dict) if reload: cnm2 = load_CNMF(fname_new[:-5] + '_cnmf_gsig.hdf5') else: # %% Extract spatial and temporal components on patches t1 = time.time() print('Starting CNMF') cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) t_patch = time.time() - t1 # %% try: dview.terminate() except: pass c, dview, n_processes = cm.cluster.setup_cluster( backend=backend_refine, n_processes=n_processes, single_thread=False) # %% if plot_on: cnm.estimates.plot_contours(img=Cn)
if np.min(images) < 0: raise Exception('Movie too negative, add_to_movie should be larger') #%% correlation image. From here infer neuron size and density Cn = cm.movie(images)[:3000].local_correlations(swap_dim=False) #%% run seeded CNMF cnm = cnmf.CNMF(n_processes, method_init='greedy_roi', k=Ain.shape[1], gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview, Ain=Ain, method_deconvolution='oasis', rolling_sum=False, rf=None) cnm = cnm.fit(images) A, C, b, f, YrA, sn = cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, cnm.sn #%% plot contours of components pl.figure() crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9) pl.title('Contour plots against correlation image') #%% evaluate the quality of the components
def test_general(): """ General Test of pipeline with comparison against ground truth A shorter version than the demo pipeline that calls comparison for the real test work Raises: --------- params_movie params_cnmf rig correction cnmf on patch cnmf full frame not able to read the file no groundtruth """ #\bug #\warning global params_movie global params_diplay fname = params_movie['fname'] niter_rig = params_movie['niter_rig'] max_shifts = params_movie['max_shifts'] splits_rig = params_movie['splits_rig'] num_splits_to_process_rig = params_movie['num_splits_to_process_rig'] cwd = os.getcwd() fname = download_demo(fname[0]) m_orig = cm.load(fname) min_mov = m_orig[:400].min() comp = comparison.Comparison() comp.dims = np.shape(m_orig)[1:] ################ RIG CORRECTION ################# t1 = time.time() mc = MotionCorrect(fname, min_mov, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, num_splits_to_process_rig=num_splits_to_process_rig, shifts_opencv=True, nonneg_movie=True) mc.motion_correct_rigid(save_movie=True) m_rig = cm.load(mc.fname_tot_rig) bord_px_rig = np.ceil(np.max(mc.shifts_rig)).astype(np.int) comp.comparison['rig_shifts']['timer'] = time.time() - t1 comp.comparison['rig_shifts']['ourdata'] = mc.shifts_rig ########################################### if 'max_shifts' not in params_movie: fnames = params_movie['fname'] border_to_0 = 0 else: # elif not params_movie.has_key('overlaps'): fnames = mc.fname_tot_rig border_to_0 = bord_px_rig m_els = m_rig idx_xy = None add_to_movie = -np.nanmin(m_els) + 1 # movie must be positive remove_init = 0 downsample_factor = 1 base_name = fname[0].split('/')[-1][:-4] name_new = cm.save_memmap_each(fnames, base_name=base_name, resize_fact=(1, 1, downsample_factor), remove_init=remove_init, idx_xy=idx_xy, add_to_movie=add_to_movie, border_to_0=border_to_0) name_new.sort() if len(name_new) > 1: fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=params_movie['n_chunks'], dview=None) else: logging.warning('One file only, not saving!') fname_new = name_new[0] Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') if np.min(images) < 0: # TODO: should do this in an automatic fashion with a while loop at the 367 line raise Exception('Movie too negative, add_to_movie should be larger') if np.sum(np.isnan(images)) > 0: # TODO: same here raise Exception( 'Movie contains nan! You did not remove enough borders') Cn = cm.local_correlations(Y) Cn[np.isnan(Cn)] = 0 p = params_movie['p'] merge_thresh = params_movie['merge_thresh'] rf = params_movie['rf'] stride_cnmf = params_movie['stride_cnmf'] K = params_movie['K'] init_method = params_movie['init_method'] gSig = params_movie['gSig'] alpha_snmf = params_movie['alpha_snmf'] if params_movie['is_dendrites'] == True: if params_movie['init_method'] is not 'sparse_nmf': raise Exception('dendritic requires sparse_nmf') if params_movie['alpha_snmf'] is None: raise Exception('need to set a value for alpha_snmf') ################ CNMF PART PATCH ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], dview=None, rf=rf, stride=stride_cnmf, memory_fact=params_movie['memory_fact'], method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=params_movie['only_init_patch'], gnb=params_movie['gnb'], method_deconvolution='oasis') comp.cnmpatch = copy.copy(cnm) comp.cnmpatch.estimates = None cnm = cnm.fit(images) A_tot = cnm.estimates.A C_tot = cnm.estimates.C YrA_tot = cnm.estimates.YrA b_tot = cnm.estimates.b f_tot = cnm.estimates.f # DISCARDING logging.info(('Number of components:' + str(A_tot.shape[-1]))) final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_patch'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_patch'] fitness_delta_min = params_movie['fitness_delta_min_patch'] Npeaks = params_movie['Npeaks'] traces = C_tot + YrA_tot idx_components, idx_components_bad = estimate_components_quality( traces, Y, A_tot, C_tot, b_tot, f_tot, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min) ####### A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1 comp.comparison['cnmf_on_patch']['ourdata'] = [A_tot.copy(), C_tot.copy()] #################### ######################## ################ CNMF PART FULL ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, p=p, Ain=A_tot, Cin=C_tot, f_in=f_tot, rf=None, stride=None, method_deconvolution='oasis') cnm = cnm.fit(images) # DISCARDING A, C, b, f, YrA, sn = cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, cnm.estimates.sn final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_full'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_full'] fitness_delta_min = params_movie['fitness_delta_min_full'] Npeaks = params_movie['Npeaks'] traces = C + YrA idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality( traces, Y, A, C, b, f, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True) ########## A_tot_full = A_tot.tocsc()[:, idx_components] C_tot_full = C_tot[idx_components] comp.comparison['cnmf_full_frame']['timer'] = time.time() - t1 comp.comparison['cnmf_full_frame']['ourdata'] = [ A_tot_full.copy(), C_tot_full.copy() ] #################### ######################## comp.save_with_compare(istruth=False, params=params_movie, Cn=Cn) log_files = glob.glob('*_LOG_*') try: for log_file in log_files: os.remove(log_file) except: logging.warning('Cannot remove log files') ############ assertions ################## pb = False if (comp.information['differences']['params_movie']): logging.error( "you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" ) pb = True if (comp.information['differences']['params_cnm']): logging.warning( "you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" ) # pb = True if (comp.information['diff']['rig']['isdifferent']): logging.error("the rigid shifts are different from the groundtruth ") pb = True if (comp.information['diff']['cnmpatch']['isdifferent']): logging.error( "the cnmf on patch produces different results than the groundtruth " ) pb = True if (comp.information['diff']['cnmfull']['isdifferent']): logging.error( "the cnmf full frame produces different results than the groundtruth " ) pb = True assert (not pb)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=border_to_0) cnm = cnm.fit(images) #%% plot contours of found components #Cn = cm.local_correlations(images.transpose(1, 2, 0)) #Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components')
def main(): pass # For compatibility between running under Spyder and the CLI c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% set up some parameters fnames = [os.path.join(caiman_datadir(), 'split', 'first3000-ch1.tif'), os.path.join(caiman_datadir(), 'split', 'second3000-ch1.tif')] is_patches = True # flag for processing in patches or not fr = 1.5 # approximate frame rate of data decay_time = 5.0 # length of transient if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 20 # half size of each patch stride = 4 # overlap between patches K = 2 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = 10 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order params_dict = {'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'rf': rf, 'stride': stride, 'K': K, 'gSig': gSig, 'merge_thr': merge_thresh, 'p': p, 'nb': gnb} opts = params.CNMFParams(params_dict=params_dict) # %% Now RUN CaImAn Batch (CNMF) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) #cnm.estimates.normalize_components() cnm = cnm.fit_file() # %% plot contour plots of components Cn = cm.load(fnames[0], subindices=slice(1000)).local_correlations(swap_dim=False) cnm.estimates.plot_contours(img=Cn) # %% load memory mapped file Yr, dims, T = cm.load_memmap(cnm.mmap_file) images = np.reshape(Yr.T, [T] + list(dims), order='F') # %% refit cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) min_SNR = 2 # peak SNR for accepted components (if above this, acept) rval_thr = 0.85 # space correlation threshold (if above this, accept) use_cnn = False # use the CNN classifier min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': use_cnn, 'min_cnn_thr': min_cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.detrend_df_f() cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% visualize selected and rejected components cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% visualize selected components cnm2.estimates.nb_view_components(images, idx=cnm2.estimates.idx_components, img=Cn) cnm2.estimates.view_components(images, idx=cnm2.estimates.idx_components_bad, img=Cn) #%% only select high quality components cnm2.estimates.select_components(use_object=True) #%% cnm2.estimates.plot_contours(img=Cn) cnm2.estimates.detrend_df_f() import pickle f = open("/home/david/zebraHorse/df_f_day55.pkl", "wb") pickle.dump(cnm2.estimates.F_dff, f) f.close() # %% play movie with results (original, reconstructed, amplified residual) for j in range(10): cnm2.estimates.play_movie(images, magnification=4.0, frame_range = range(100 * j, 100 * (j + 1))) #import time #time.sleep(1000) # %% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file)
def find_rois_cnmf(video_path, params, mc_borders=None, use_multiprocessing=True): full_video_path = video_path directory = os.path.dirname(full_video_path) filename = os.path.basename(full_video_path) memmap_video = tifffile.memmap(video_path) print(memmap_video.shape) if len(memmap_video.shape) == 5: num_z = memmap_video.shape[2] else: num_z = memmap_video.shape[1] roi_spatial_footprints = [ None for i in range(num_z) ] roi_temporal_footprints = [ None for i in range(num_z) ] roi_temporal_residuals = [ None for i in range(num_z) ] bg_spatial_footprints = [ None for i in range(num_z) ] bg_temporal_footprints = [ None for i in range(num_z) ] # Create the cluster if use_multiprocessing: if os.name == 'nt': backend = 'multiprocessing' else: backend = 'ipyparallel' cm.stop_server() c, dview, n_processes = cm.cluster.setup_cluster(backend=backend, n_processes=None, single_thread=False) else: dview = None for z in range(num_z): fname = os.path.splitext(filename)[0] + "_masked_z_{}.tif".format(z) video_path = os.path.join(directory, fname) if len(memmap_video.shape) == 5: tifffile.imsave(video_path, memmap_video[:, :, z, :, :].reshape((-1, memmap_video.shape[3], memmap_video.shape[4]))) else: tifffile.imsave(video_path, memmap_video[:, z, :, :]) # dataset dependent parameters fnames = [video_path] # filename to be processed fr = params['imaging_fps'] # imaging rate in frames per second decay_time = params['decay_time'] # length of a typical transient in seconds # parameters for source extraction and deconvolution p = params['autoregressive_order'] # order of the autoregressive system gnb = params['num_bg_components'] # number of global background components merge_thresh = params['merge_threshold'] # merging threshold, max correlation allowed rf = None # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = params['num_components'] # number of components per patch gSig = [params['half_size'], params['half_size']] # expected half size of neurons init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') rolling_sum = True rolling_length = 50 is_dendrites = False # flag for analyzing dendritic data alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF # parameters for component evaluation min_SNR = params['min_snr'] # signal to noise ratio for accepting a component rval_thr = params['min_spatial_corr'] # space correlation threshold for accepting a component cnn_thr = params['cnn_threshold'] # threshold for CNN based classifier if mc_borders is not None: border_pix = mc_borders[z] else: border_pix = 0 fname_new = cm.save_memmap(fnames, base_name='memmap_z_{}'.format(z), order='C') # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh= merge_thresh, p = p, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, rolling_sum=rolling_sum, only_init_patch = False, gnb = gnb, border_pix = border_pix, ssub=1, ssub_B=1, tsub=1) cnm = cnm.fit(images) try: A_in, C_in, b_in, f_in = cnm.A, cnm.C, cnm.b, cnm.f except: A_in, C_in, b_in, f_in = cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in = b_in, f_in=f_in, rf = None, stride = None, gnb = gnb, method_deconvolution='oasis', check_nan = True) cnm2 = cnm2.fit(images) try: roi_spatial_footprints[z] = cnm2.A roi_temporal_footprints[z] = cnm2.C roi_temporal_residuals[z] = cnm2.YrA bg_spatial_footprints[z] = cnm2.b bg_temporal_footprints[z] = cnm2.f except: roi_spatial_footprints[z] = cnm2.estimates.A roi_temporal_footprints[z] = cnm2.estimates.C roi_temporal_residuals[z] = cnm2.estimates.YrA bg_spatial_footprints[z] = cnm2.estimates.b bg_temporal_footprints[z] = cnm2.estimates.f if os.path.exists(video_path): os.remove(video_path) del memmap_video if use_multiprocessing: if backend == 'multiprocessing': dview.close() else: try: dview.terminate() except: dview.shutdown() cm.stop_server() mmap_files = glob.glob(os.path.join(directory, "*.mmap")) for mmap_file in mmap_files: try: os.remove(mmap_file) except: pass log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file) return roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints