def _load_data(self): self.current_sample_id = None self.ix = 0 dfof_curves = None proj_path = self.t.get_proj_path() self.t.df['_DETREND_DF_O_F'] = [np.array([], dtype=np.float64)] * self.t.df.index.size for index, row in self.t.df.iterrows(): self.ix += 1 if row['SampleID'] != self.current_sample_id: self.idx_components = None self.ix = 0 dfof_curves = None self.current_sample_id = row['SampleID'] pikPath = os.path.join(proj_path, row['ImgInfoPath']) pik = pickle.load(open(pikPath, 'rb')) cnmA = pik['roi_states']['cnmf_output']['cnmA'] cnmb = pik['roi_states']['cnmf_output']['cnmb'] cnmC = pik['roi_states']['cnmf_output']['cnmC'] cnm_f = pik['roi_states']['cnmf_output']['cnm_f'] cnmYrA = pik['roi_states']['cnmf_output']['cnmYrA'] self.idx_components = pik['roi_states']['cnmf_output']['idx_components'] try: dfof_curves = detrend_df_f(cnmA, cnmb, cnmC, cnm_f, YrA=cnmYrA, quantileMin=self.params['quantileMin'], frames_window=self.params['frames_window']) # , # self.dfof_curves = dfof_curves except: tb = traceback.format_exc() raise Exception('The following exception was raised in caiman detrend_df_f method for SampleID: ' + self.current_sample_id + '\n\n' + tb) # flag_auto=self.ctrls['auto_quantile'].isChecked(), # use_fast=self.ctrls['fast_filter'].isChecked()) if self.idx_components[self.ix] != row['ROI_State']['cnmf_idx']: raise Exception( 'CNMF component index Mismatch Error! Something went very wrong. Check the indices of your ' 'CNMF components from SampleID: ' + self.current_sample_id + '\nIndex in pickle is: ' + str(self.idx_components[self.ix]) + '\nIndex in dataframe is: ' + str(row['ROI_State']['cnmf_idx']) + '\n at iloc: ' + str(index)) self.t.df.at[index, '_DETREND_DF_O_F'] = dfof_curves[self.idx_components[self.ix]]
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set this to true to show movies and plots fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. To # close the video press q m_orig = cm.load_movie_chain(fname[:1]) downsample_ratio = 0.2 offset_mov = -np.min(m_orig[:100]) moviehandle = m_orig.resize(1, 1, downsample_ratio) if display_images: moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # maximum shift to be used for trimming against NaNs #%% compare with original movie moviehandle = cm.concatenate([ m_orig.resize(1, 1, downsample_ratio) + offset_mov, m_els.resize(1, 1, downsample_ratio) ], axis=2) display_images = False if display_images: moviehandle.play(fr=60, q_max=99.5, magnification=2, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) #%% PLOT COMPONENTS if display_images: plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) if display_images: view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=Cn) view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components_bad], cnm.C[idx_components_bad], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components_bad], img=Cn) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape( dims + (-1, ), order='F').transpose([2, 0, 1]) #%% play along side original data moviehandle = cm.concatenate([ m_els.resize(1, 1, downsample_ratio), denoised.resize(1, 1, downsample_ratio) ], axis=2) if display_images: moviehandle.play(fr=60, gain=15, magnification=2, offset=0) # press q to exit
idx_components_bad=idx_components_bad, SNR_comp=SNR_comp, cnn_preds=cnn_preds, r_values=r_values) #%% plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm2.A[:, idx_components], Cn, thr=.96, vmax=0.5) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours(cnm2.A[:, idx_components_bad], Cn, thr=.96, vmax=0.5) plt.title('Contour plots of rejected components') #%% #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A[:, idx_components], cnm2.b, cnm2.C[idx_components], cnm2.f, YrA=cnm2.YrA[idx_components], quantileMin=8, frames_window=250)
b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + \
plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm2.A[:, idx_components_bad], Cn, thr=.95, vmax=0.25) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) view_patches_bar(Yr, cnm2.A.tocsc()[:, idx_components], cnm2.C[idx_components], cnm2.b, cnm2.f, dims[0], dims[1], YrA=cnm2.YrA[idx_components], img=Cn) view_patches_bar(Yr, cnm2.A.tocsc()[:, idx_components_bad], cnm2.C[idx_components_bad], cnm2.b, cnm2.f, dims[0], dims[1], YrA=cnm2.YrA[idx_components_bad], img=Cn) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape(dims + (-1,), order='F').transpose([2, 0, 1]) #%% play along side original data
big_string = '' for substring in my_list: big_string = big_string + ' ' + substring A = pcf.cnmf.estimates.A F = pcf.cnmf.estimates.C + pcf.cnmf.estimates.YrA b = pcf.cnmf.estimates.b f= pcf.cnmf.estimates.f B = A.T.dot(b).dot(f) import scipy.ndimage as nd Df = nd.percentile_filter(B, 10, (1000,1)) plt.figure(); plt.plot(B[49]+pcf.cnmf.estimates.C[49]+pcf.cnmf.estimates.R[49]) #%% Flag auto as False, how does dFF look import caiman.source_extraction.cnmf.utilities as ut flag_dff = ut.detrend_df_f(A, b, pcf.cnmf.estimates.C, f, YrA=pcf.cnmf.estimates.YrA, quantileMin=8, frames_window=500, flag_auto=False, use_fast=False, detrend_only=False) slow_dff = ut.extract_DF_F(Yr, A, C, bl, quantileMin=8, frames_window=200, block_size=400, dview=None) fig, ax = plt.subplots(2) ax[0].plot(pcf.cnmf.estimates.F_dff[49]) ax[1].plot(flag_dff[49]) plt.figure() plt.plot(flag_dff[49], label='auto=False') plt.plot(pcf.cnmf.estimates.F_dff[49], label='auto=True') plt.legend()
def caiman_main_light_weight(fr, fnames, z=0, dend=False): """ Main function to compute the caiman algorithm. For more details see github and papers fpath(str): Folder where to store the plots fr(int): framerate fnames(list-str): list with the names of the files to be computed together z(array): vector with the values of z relative to y dend(bool): Boleean to change parameters to look for neurons or dendrites display_images(bool): to display and save different plots returns F_dff(array): array with the dff of the components com(array): matrix with the position values of the components as given by caiman cnm(struct): struct with different stimates and returns from caiman""" # parameters decay_time = 0.4 # length of a typical transient in seconds # Look for the best parameters for this 2p system and never change them again :) # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (3, 3) # maximum allow rigid shift splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time strides = ( 96, 96 ) # start a new patch for pw-rigid motion correction every x pixels overlaps = (48, 48 ) # overlap between pathes (size of patch strides+overlaps) splits_els = 10 # for parallelization split the movies in num_splits chuncks across time upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches max_deviation_rigid = 3 # maximum deviation allowed for patch with respect to rigid shifts # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed rf = 25 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 10 # amount of overlap between the patches in pixels K = 25 # number of components per patch if dend: gSig = [1, 1] # expected half size of neurons init_method = 'sparse_nmf' # initialization method (if analyzing dendritic data using 'sparse_nmf') alpha_snmf = 1e-6 # sparsity penalty for dendritic data analysis through sparse NMF else: gSig = [3, 3] # expected half size of neurons init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier dview = None # parallel processing keeps crashing. print('***************Starting motion correction*************') print('files:') print(fnames) # %% start a cluster for parallel processing # c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fnames[0]).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fnames, min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory # %% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) totdes = [np.nansum(mc.x_shifts_els), np.nansum(mc.y_shifts_els)] print('***************Motion correction has ended*************') # maximum shift to be used for trimming against NaNs # %% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory # cm.stop_server(dview=dview) # c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% RUN CNMF ON PATCHES print('***************Running CNMF...*************') # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.estimates.A[:, idx_components], cnm.estimates.C[ idx_components], cnm.estimates.b, cnm.estimates.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) print('***************Fit*************') cnm2 = cnm2.fit(images) print('***************Extractind DFFs*************') # %% Extract DF/F values # cm.stop_server(dview=dview) try: F_dff = detrend_df_f(cnm2.estimates.A, cnm2.estimates.b, cnm2.estimates.C, cnm2.estimates.f, YrA=cnm2.estimates.YrA, quantileMin=8, frames_window=250) # F_dff = detrend_df_f(cnm.A, cnm.b, cnm.C, cnm.f, YrA=cnm.YrA, quantileMin=8, frames_window=250) except: F_dff = cnm2.estimates.C * np.nan print('WHAAT went wrong again?') print('***************stopping cluster*************') # %% STOP CLUSTER and clean up log files # cm.stop_server(dview=dview) # *************************************************************************************** # Preparing output data # F_dff -> DFF values, is a matrix [number of neurons, length recording] # com --> center of mass, is a matrix [number of neurons, 2] print('***************preparing output data*************') if len(dims) <= 2: if len(z) == 1: com = np.concatenate( (cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]), np.zeros((cnm2.estimates.A.shape[1], 1)) + z), 1) elif len(z) == dims[0]: auxcom = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]) zy = np.zeros((auxcom.shape[0], 1)) for y in np.arange(auxcom.shape[0]): zy[y, 0] = z[int(auxcom[y, 0])] com = np.concatenate((auxcom, zy), 1) else: print( 'WARNING: Z value was not correctly defined, only X and Y values on file, z==zeros' ) print(['length of z was: ' + str(len(z))]) com = np.concatenate( (cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]), np.zeros((cnm2.estimates.A.shape[1], 1))), 1) else: com = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1], dims[2]) return F_dff, com, cnm2, totdes, SNR_comp[idx_components]