def extract(mmap_file, cnmf_opts, nprocs=8, dview=None): yr, dims, t = cm.load_memmap(mmap_file) y = yr.T.reshape((t, *dims), order='F') cnmf = source.cnmf.CNMF(n_processes=nprocs, method_init=cnmf_opts['init-method'], gSiz=[cnmf_opts['gsiz']] * 2, gSig=[cnmf_opts['gsig']] * 2, merge_thresh=cnmf_opts['merge-thresh'], Ain=None, k=None, tsub=cnmf_opts['t-sub'], ssub=cnmf_opts['s-sub'], rf=cnmf_opts['half-patch-size'], p=1, dview=dview, stride=cnmf_opts['patch-overlap'], only_init_patch=True, method_deconvolution='oasis', nb_patch=-1, gnb=cnmf_opts['background-components'], low_rank_background=None, update_background_components=True, min_corr=cnmf_opts['min-corr'], min_pnr=cnmf_opts['min-pnr'], normalize_init=False, center_psf=True, ring_size_factor=cnmf_opts['ring-size-factor'], del_duplicates=True, ssub_B=2, border_pix=cnmf_opts['border-px']) cnmf.fit(y) estimates = cnmf.estimates good_idx, bad_idx, _, _, _ = evaluate.estimate_components_quality_auto( Y=y, A=estimates.A, C=estimates.C, b=estimates.b, f=estimates.f, YrA=estimates.YrA, frate=cnmf_opts['fps'], decay_time=cnmf_opts['decay-time'], gSig=cnmf_opts['gsig'], dims=dims, dview=None, min_SNR=cnmf_opts['min-snr'], use_cnn=False) return estimates.C[good_idx], estimates.A.toarray()[:, good_idx], cnmf, dims
def components_eval(images, cnm, dview, dims): '''Evaluating the components being found Args: images: The input video shape: (frames, dims, dims) cnm: Object obtained from CNMF algorithm with C,A,S,b,f dview: Direct View object for parallelization pruposes when using ipyparallel dims: Dimension of each frame shape: (521, 521) Returns: idx_components: list of ids of components that are considered to be a neuron ''' idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = estimate_components_quality_auto( images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=use_cnn, thresh_cnn_lowest=min_cnn_thr) return idx_components
def main(): pass # For compatibility between running under Spyder and the CLI #%% start a cluster c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% save files to be processed # This datafile is distributed with Caiman fnames = [ os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif') ] # location of dataset (can actually be a list of filed to be concatenated) add_to_movie = -np.min(cm.load(fnames[0], subindices=range(200))).astype(float) # determine minimum value on a small chunk of data add_to_movie = np.maximum(add_to_movie, 0) # if minimum is negative subtract to make the data non-negative base_name = 'Yr' fname_new = cm.save_memmap(fnames, dview=dview, base_name=base_name, order='C', add_to_movie=add_to_movie) #%% LOAD MEMORY MAPPABLE FILE Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') #%% play movie, press q to quit play_movie = False if play_movie: cm.movie(images[1400:]).play(fr=50, magnification=4, gain=3.) #%% correlation image. From here infer neuron size and density Cn = cm.movie(images).local_correlations(swap_dim=False) plt.imshow(Cn, cmap='gray') plt.title('Correlation Image') #%% set up some parameters is_patches = True # flag for processing in patches or not if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 10 # half size of each patch stride = 4 # overlap between patches K = 4 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = 30 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order #%% Now RUN CNMF cnm = cnmf.CNMF(n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview, gnb=gnb, rf=rf, stride=stride, rolling_sum=False) cnm = cnm.fit(images) #%% plot contour plots of components plt.figure() crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of components') #%% A_in, C_in, b_in, f_in = cnm.A[:, :], cnm.C[:], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) fr = 10 # approximate frame rate of data decay_time = 5.0 # length of transient min_SNR = 2.5 # peak SNR for accepted components (if above this, acept) rval_thr = 0.90 # space correlation threshold (if above this, accept) use_cnn = True # use the CNN classifier min_cnn_thr = 0.95 # if cnn classifier predicts below this value, reject idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=use_cnn, thresh_cnn_min=min_cnn_thr) #%% visualize selected and rejected components plt.figure() plt.subplot(1, 2, 1) cm.utils.visualization.plot_contours(cnm2.A[:, idx_components], Cn, thr=0.9) plt.title('Selected components') plt.subplot(1, 2, 2) plt.title('Discaded components') cm.utils.visualization.plot_contours(cnm2.A[:, idx_components_bad], Cn, thr=0.9) #%% plt.figure() crd = cm.utils.visualization.plot_contours(cnm2.A.tocsc()[:, idx_components], Cn, thr=0.9) plt.title('Contour plots of components') #%% visualize selected components cm.utils.visualization.view_patches_bar(Yr, cnm2.A.tocsc()[:, idx_components], cnm2.C[idx_components, :], cnm2.b, cnm2.f, dims[0], dims[1], YrA=cnm2.YrA[idx_components, :], img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server() log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file)
def filter_rois(video_paths, roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints, params): directory = os.path.dirname(video_paths[0]) final_video_path = os.path.join(directory, "final_video_temp.tif") with tifffile.TiffWriter(final_video_path, bigtiff=True) as tif: for i in range(len(video_paths)): video_path = video_paths[i] video = tifffile.memmap(video_path) if len(video.shape) == 3: # add a z dimension video = video[:, np.newaxis, :, :] tif.save(video) del video final_video = tifffile.memmap(final_video_path).astype(np.uint16) if len(final_video.shape) == 5: final_video_path_2 = os.path.join(directory, "final_video_temp_2.tif") tifffile.imsave(final_video_path_2, final_video.reshape((final_video.shape[0]*final_video.shape[1], final_video.shape[2], final_video.shape[3], final_video.shape[4]))) if os.path.exists(final_video_path): os.remove(final_video_path) else: final_video_path_2 = final_video_path filtered_out_rois = [] memmap_video = tifffile.memmap(final_video_path_2) if len(memmap_video.shape) == 5: n_frames = memmap_video.shape[1] num_z = memmap_video.shape[2] height = memmap_video.shape[3] width = memmap_video.shape[4] else: n_frames = memmap_video.shape[0] num_z = memmap_video.shape[1] height = memmap_video.shape[2] width = memmap_video.shape[3] for z in range(num_z): directory = os.path.dirname(final_video_path_2) filename = os.path.basename(final_video_path_2) fname = os.path.splitext(filename)[0] + "_masked_z_{}_d1_{}_d2_{}_d3_1_order_C_frames_{}_.mmap".format(z, height, width, n_frames) video_path = os.path.join(directory, fname) if len(memmap_video.shape) == 5: tifffile.imsave(video_path, memmap_video[:, :, z, :, :].reshape((-1, memmap_video.shape[3], memmap_video.shape[4])).transpose([1, 2, 0]).reshape(-1, memmap_video.shape[0]).astype(np.float32)) else: tifffile.imsave(video_path, memmap_video[:, z, :, :].transpose([1, 2, 0]).reshape(-1, memmap_video.shape[0]).astype(np.float32)) video = tifffile.memmap(video_path) print(video.shape) idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(video, roi_spatial_footprints[z], roi_temporal_footprints[z], bg_spatial_footprints[z], bg_temporal_footprints[z], roi_temporal_residuals[z], params['imaging_fps']/num_z, params['decay_time'], params['half_size'], (video.shape[-2], video.shape[-1]), dview = None, min_SNR=params['min_snr'], r_values_min = params['min_spatial_corr'], use_cnn = params['use_cnn'], thresh_cnn_lowest = params['cnn_threshold'], gSig_range=[ (i, i) for i in range(max(1, params['half_size']-5), params['half_size']+5) ]) filtered_out_rois.append(list(idx_components_bad)) if isinstance(roi_spatial_footprints[z], scipy.sparse.coo_matrix): f = roi_spatial_footprints[z].toarray() else: f = roi_spatial_footprints[z] for i in range(f.shape[-1]): area = np.sum(f[:, i] > 0) if (area < params['min_area'] or area > params['max_area']) and i not in filtered_out_rois[-1]: filtered_out_rois[-1].append(i) if os.path.exists(video_path): try: os.remove(video_path) except: pass del memmap_video if os.path.exists(final_video_path_2): os.remove(final_video_path_2) mmap_files = glob.glob(os.path.join(directory, "*.mmap")) for mmap_file in mmap_files: try: os.remove(mmap_file) except: pass log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file) return filtered_out_rois
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set this to true to show movies and plots fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. To # close the video press q m_orig = cm.load_movie_chain(fname[:1]) downsample_ratio = 0.2 offset_mov = -np.min(m_orig[:100]) moviehandle = m_orig.resize(1, 1, downsample_ratio) if display_images: moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # maximum shift to be used for trimming against NaNs #%% compare with original movie moviehandle = cm.concatenate([ m_orig.resize(1, 1, downsample_ratio) + offset_mov, m_els.resize(1, 1, downsample_ratio) ], axis=2) display_images = False if display_images: moviehandle.play(fr=60, q_max=99.5, magnification=2, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) #%% PLOT COMPONENTS if display_images: plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) if display_images: view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=Cn) view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components_bad], cnm.C[idx_components_bad], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components_bad], img=Cn) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape( dims + (-1, ), order='F').transpose([2, 0, 1]) #%% play along side original data moviehandle = cm.concatenate([ m_els.resize(1, 1, downsample_ratio), denoised.resize(1, 1, downsample_ratio) ], axis=2) if display_images: moviehandle.play(fr=60, gain=15, magnification=2, offset=0) # press q to exit
cnm = cnm.fit(images) t_refine = time.time() - t1 A, C, b, f, YrA, sn = cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, cnm.sn # %% again recheck quality of components, stricter criteria idx_components, idx_components_bad, comp_SNR, comp_SNR_delta, r_values, predictionsCNN = estimate_components_quality_auto( Y, A, C, b, f, YrA, params_movie['fr'], params_movie['decay_time'], gSig, dims, dview=dview, min_SNR=global_params['min_SNR'], r_values_min=global_params['rval_thr'], r_values_lowest=global_params['min_rval_thr_rejected'], Npeaks=global_params['Npeaks'], use_cnn=True, thresh_cnn_min=global_params['min_cnn_thresh'], thresh_cnn_lowest=global_params[ 'max_classifier_probability_rejected'], thresh_fitness_delta=global_params['max_fitness_delta_accepted']) # N_samples = np.ceil(params_movie['fr']*params_movie['decay_time']).astype(np.int) # number of timesteps to consider when testing new neuron candidates # min_SNR = global_params['min_SNR'] # adaptive way to set threshold (will be equal to min_SNR) # #pr_inc = 1 - scipy.stats.norm.cdf(global_params['min_SNR']) # inclusion probability of noise transient # #thresh_fitness_raw = np.log(pr_inc)*N_samples # event exceptionality threshold
#%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier # parameters for component evaluation min_SNR = 1 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.5 # threshold for CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm2.A, cnm2.C, cnm2.b, cnm2.f, cnm2.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr, gSig_range = [list(np.add(i*2,a)) for i,a in enumerate([gSig]*5)]) #%% PLOT COMPONENTS plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours( cnm2.A[:, idx_components], Cn, thr=.95, vmax=0.25) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm2.A[:, idx_components_bad], Cn, thr=.95, vmax=0.25) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected)
# roi_all_match = roi_1_match#np.concatenate([roi_all_match, roi_1_match],axis = 0) A_gt_thr = roi_all_match.transpose([1, 2, 0]).reshape( (np.prod(dims), -1), order='F') A_gt = A_gt_thr else: if filter_SNR: final_frate = params_movie['fr'] Npeaks = global_params['Npeaks'] traces_gt = C_gt + YrA_gt idx_filter_snr, idx_filter_snr_bad, fitness_raw_gt, fitness_delta_gt, r_values_gt = estimate_components_quality_auto( traces_gt, Y, A_gt, C_gt, b_gt, f_gt, final_frate=final_frate, Npeaks=Npeaks, r_values_min=1, fitness_min=-10, fitness_delta_min=-10, return_all=True) print(len(idx_filter_snr)) print(len(idx_filter_snr_bad)) A_gt, C_gt, b_gt, f_gt, traces_gt, YrA_gt = A_gt.tocsc( )[:, idx_filter_snr], C_gt[idx_filter_snr], b_gt, f_gt, traces_gt[ idx_filter_snr], YrA_gt[idx_filter_snr] A_gt_thr = cm.source_extraction.cnmf.spatial.threshold_components(
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set to true to show movies and images fnames = ['data_endoscope.tif'] # filename to be processed frate = 10 # movie frame rate decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters do_motion_correction_nonrigid = True do_motion_correction_rigid = False # in this case it will also save a rigid motion corrected movie gSig_filt = (3, 3) # size of filter, in general gSig (see below), # change this one if algorithm does not work max_shifts = (5, 5) # maximum allowed rigid shift splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time strides = (48, 48) # start a new patch for pw-rigid motion correction every x pixels overlaps = (24, 24) # overlap between pathes (size of patch strides+overlaps) # for parallelization split the movies in num_splits chuncks across time # (remember that it should hold that length_movie/num_splits_to_process_rig>100) splits_els = 10 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None gSig = 3 # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = 13 # average diameter of a neuron, in general 4*gSig+1 merge_thresh = .7 # merging threshold, max correlation allowed rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80 stride_cnmf = 20 # amount of overlap between the patches in pixels # (keep it at least large as gSiz, i.e 4 times the neuron size gSig) tsub = 2 # downsampling factor in time for initialization, # increase if you have memory problems ssub = 1 # downsampling factor in space for initialization, # increase if you have memory problems Ain = None # if you want to initialize with some preselected components # you can pass them here as boolean vectors low_rank_background = None # None leaves background of each patch intact, # True performs global low-rank approximation gnb = -1 # number of background components (rank) if positive, # else exact ring model with following settings # gnb=-2: Return background as b and W # gnb=-1: Return full rank background B # gnb= 0: Don't return background nb_patch = -1 # number of background components (rank) per patch, # use 0 or -1 for exact background of ring model (cf. gnb) min_corr = .8 # min peak value from correlation image min_pnr = 10 # min peak to noise ration from PNR image ssub_B = 2 # additional downsampling factor in space for background ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor # parameters for component evaluation min_SNR = 3 # adaptive way to set threshold on the transient size r_values_min = 0.85 # threshold on space consistency (if you lower more components # will be accepted, potentially with worst quality) #%% start the cluster try: cm.stop_server(dview=dview) # stop it if it was running except: pass c, dview, n_processes = cm.cluster.setup_cluster(backend='local', # use this one n_processes=24, # number of process to use, if you go out of memory try to reduce this one single_thread=False) #%% download demo file fnames = [download_demo(fnames[0])] filename_reorder = fnames #%% MOTION CORRECTION if do_motion_correction_nonrigid or do_motion_correction_rigid: # do motion correction rigid mc = motion_correct_oneP_rigid(fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, dview=dview, splits_rig=splits_rig, save_movie=not(do_motion_correction_nonrigid) ) new_templ = mc.total_template_rig plt.subplot(1, 2, 1) plt.imshow(new_templ) # % plot template plt.subplot(1, 2, 2) plt.plot(mc.shifts_rig) # % plot rigid shifts plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames') plt.ylabel('pixels') # borders to eliminate from movie because of motion correction bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) filename_reorder = mc.fname_tot_rig # do motion correction nonrigid if do_motion_correction_nonrigid: mc = motion_correct_oneP_nonrigid( fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, dview=dview, splits_rig=None, save_movie=True, # whether to save movie in memory mapped format new_templ=new_templ # template to initialize motion correction ) filename_reorder = mc.fname_tot_els bord_px = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # create memory mappable file in the right order on the hard drive (C order) fname_new = cm.save_memmap_each( filename_reorder, base_name='memmap_', order='C', border_to_0=bord_px, dview=dview) fname_new = cm.save_memmap_join(fname_new, base_name='memmap_', dview=dview) # load memory mappable file Yr, dims, T = cm.load_memmap(fname_new) Y = Yr.T.reshape((T,) + dims, order='F') #%% compute some summary images (correlation and peak to noise) # change swap dim if output looks weird, it is a problem with tiffile cn_filter, pnr = cm.summary_images.correlation_pnr(Y, gSig=gSig, swap_dim=False) # inspect the summary images and set the parameters inspect_correlation_pnr(cn_filter, pnr) # print parameters set above, modify them if necessary based on summary images print(min_corr) # min correlation of peak (from correlation image) print(min_pnr) # min peak to noise ratio #%% RUN CNMF ON PATCHES cnm = cnmf.CNMF( n_processes=n_processes, method_init='corr_pnr', # use this for 1 photon k=K, gSig=(gSig, gSig), gSiz=(gSiz, gSiz), merge_thresh=merge_thresh, p=p, dview=dview, tsub=tsub, ssub=ssub, Ain=Ain, rf=rf, stride=stride_cnmf, only_init_patch=True, # just leave it as is gnb=gnb, nb_patch=nb_patch, method_deconvolution='oasis', # could use 'cvxpy' alternatively low_rank_background=low_rank_background, update_background_components=True, # sometimes setting to False improve the results min_corr=min_corr, min_pnr=min_pnr, normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon ssub_B=ssub_B, ring_size_factor=ring_size_factor, del_duplicates=True, # whether to remove duplicates from initialization border_pix=bord_px) # number of pixels to not consider in the borders cnm.fit(Y) #%% DISCARD LOW QUALITY COMPONENTS idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = \ estimate_components_quality_auto( Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False) print(' ***** ') print((len(cnm.C))) print((len(idx_components))) cm.stop_server(dview=dview) #%% PLOT COMPONENTS if display_images: plt.figure(figsize=(12, 6)) plt.subplot(121) crd_good = cm.utils.visualization.plot_contours( cnm.A[:, idx_components], cn_filter, thr=.8, vmax=0.95) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], cn_filter, thr=.8, vmax=0.95) plt.title('Contour plots of rejected components') #%% VISUALIZE IN DETAILS COMPONENTS cm.utils.visualization.view_patches_bar( Yr, cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=cn_filter) #%% MOVIES if display_images: B = cnm.b.dot(cnm.f) if 'sparse' in str(type(B)): B = B.toarray() # denoised movie cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot(cnm.C[idx_components]) + B, dims + (-1,), order='F').transpose(2, 0, 1)).play(magnification=3, gain=1.) # only neurons cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot( cnm.C[idx_components]), dims + (-1,), order='F').transpose(2, 0, 1) ).play(magnification=3, gain=10.) # only the background cm.movie(np.reshape(B, dims + (-1,), order='F').transpose(2, 0, 1) ).play(magnification=3, gain=1.) # residuals cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B, dims + (-1,), order='F').transpose(2, 0, 1) ).play(magnification=3, gain=10., fr=10) # eventually, you can rerun the algorithm on the residuals plt.imshow(cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B, dims + (-1,), order='F').transpose(2, 0, 1) ).local_correlations(swap_dim=False))
# the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) fr = 3 # approximate frame rate of data decay_time = 1 # length of transient min_SNR = 2.5 # peak SNR for accepted components (if above this, acept) rval_thr = 0.90 # space correlation threshold (if above this, accept) use_cnn = False # use the CNN classifier min_cnn_thr = 0.95 # if cnn classifier predicts below this value, reject idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm2.A, cnm2.C, cnm2.b, cnm2.f, cnm2.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=use_cnn, thresh_cnn_min=min_cnn_thr, thresh_cnn_lowest=0) #%% cm.utils.visualization.plot_contours(cnm2.A, Cn, thr=0.9) #%% visualize selected and rejected components plt.figure() plt.subplot(1, 2, 1) cm.utils.visualization.plot_contours(cnm2.A[:, idx_components], Cn, thr=0.9) plt.title('Selected components') plt.subplot(1, 2, 2) plt.title('Discaded components') cm.utils.visualization.plot_contours(cnm2.A[:, idx_components_bad], Cn, thr=0.9)
min_pnr=min_pnr, # min peak to noise ration from PNR image normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon del_duplicates=True) # whether to remove duplicates from initialization cnm.fit(Y) # %% DISCARD LOW QUALITY COMPONENTS idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = estimate_components_quality_auto( Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False) print(' ***** ') print((len(cnm.C))) print((len(idx_components))) #%% PLOT ALL COMPONENTS crd = cm.utils.visualization.plot_contours(cnm.A, cn_filter, thr=.8, vmax=0.95) #%% PLOT ONLY GOOD QUALITY COMPONENTS crd = cm.utils.visualization.plot_contours(cnm.A.tocsc()[:, idx_components],
def caiman_main_light_weight(fr, fnames, z=0, dend=False): """ Main function to compute the caiman algorithm. For more details see github and papers fpath(str): Folder where to store the plots fr(int): framerate fnames(list-str): list with the names of the files to be computed together z(array): vector with the values of z relative to y dend(bool): Boleean to change parameters to look for neurons or dendrites display_images(bool): to display and save different plots returns F_dff(array): array with the dff of the components com(array): matrix with the position values of the components as given by caiman cnm(struct): struct with different stimates and returns from caiman""" # parameters decay_time = 0.4 # length of a typical transient in seconds # Look for the best parameters for this 2p system and never change them again :) # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (3, 3) # maximum allow rigid shift splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time strides = ( 96, 96 ) # start a new patch for pw-rigid motion correction every x pixels overlaps = (48, 48 ) # overlap between pathes (size of patch strides+overlaps) splits_els = 10 # for parallelization split the movies in num_splits chuncks across time upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches max_deviation_rigid = 3 # maximum deviation allowed for patch with respect to rigid shifts # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed rf = 25 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 10 # amount of overlap between the patches in pixels K = 25 # number of components per patch if dend: gSig = [1, 1] # expected half size of neurons init_method = 'sparse_nmf' # initialization method (if analyzing dendritic data using 'sparse_nmf') alpha_snmf = 1e-6 # sparsity penalty for dendritic data analysis through sparse NMF else: gSig = [3, 3] # expected half size of neurons init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier dview = None # parallel processing keeps crashing. print('***************Starting motion correction*************') print('files:') print(fnames) # %% start a cluster for parallel processing # c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fnames[0]).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fnames, min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory # %% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) totdes = [np.nansum(mc.x_shifts_els), np.nansum(mc.y_shifts_els)] print('***************Motion correction has ended*************') # maximum shift to be used for trimming against NaNs # %% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory # cm.stop_server(dview=dview) # c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% RUN CNMF ON PATCHES print('***************Running CNMF...*************') # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.estimates.A[:, idx_components], cnm.estimates.C[ idx_components], cnm.estimates.b, cnm.estimates.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) print('***************Fit*************') cnm2 = cnm2.fit(images) print('***************Extractind DFFs*************') # %% Extract DF/F values # cm.stop_server(dview=dview) try: F_dff = detrend_df_f(cnm2.estimates.A, cnm2.estimates.b, cnm2.estimates.C, cnm2.estimates.f, YrA=cnm2.estimates.YrA, quantileMin=8, frames_window=250) # F_dff = detrend_df_f(cnm.A, cnm.b, cnm.C, cnm.f, YrA=cnm.YrA, quantileMin=8, frames_window=250) except: F_dff = cnm2.estimates.C * np.nan print('WHAAT went wrong again?') print('***************stopping cluster*************') # %% STOP CLUSTER and clean up log files # cm.stop_server(dview=dview) # *************************************************************************************** # Preparing output data # F_dff -> DFF values, is a matrix [number of neurons, length recording] # com --> center of mass, is a matrix [number of neurons, 2] print('***************preparing output data*************') if len(dims) <= 2: if len(z) == 1: com = np.concatenate( (cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]), np.zeros((cnm2.estimates.A.shape[1], 1)) + z), 1) elif len(z) == dims[0]: auxcom = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]) zy = np.zeros((auxcom.shape[0], 1)) for y in np.arange(auxcom.shape[0]): zy[y, 0] = z[int(auxcom[y, 0])] com = np.concatenate((auxcom, zy), 1) else: print( 'WARNING: Z value was not correctly defined, only X and Y values on file, z==zeros' ) print(['length of z was: ' + str(len(z))]) com = np.concatenate( (cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]), np.zeros((cnm2.estimates.A.shape[1], 1))), 1) else: com = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1], dims[2]) return F_dff, com, cnm2, totdes, SNR_comp[idx_components]
def CNMF_PROCESS(tif_movie, _k, _g, _merge): """ Inputs .tif movie (transforming from time series images) Applys constrained nonnegative matrix factorization Outputs selected neurons sparse matrix and dimension of movie """ dataset_name = tif_movie.replace('.tif', '') # start a cluster c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # process movie file fnames = [tif_movie] # location of dataset (can actually be a list of filed to be concatenated) add_to_movie = -np.min(cm.load(fnames[0], subindices=range(200))).astype(float) # determine minimum value on a small chunk of data add_to_movie = np.maximum(add_to_movie, 0) # if minimum is negative subtract to make the data non-negative base_name = 'Yr' name_new = cm.save_memmap_each(fnames, dview=dview, base_name=base_name, add_to_movie=add_to_movie) name_new.sort() fname_new = cm.save_memmap_join(name_new, base_name='Yr', dview=dview) ### LOAD MEMORY MAPPABLE FILE Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # ### play movie, press q to quit # play_movie = False # if play_movie: # cm.movie(images[1400:]).play(fr=50, magnification=4, gain=3.) ### correlation image. From here infer neuron size and density Cn = cm.movie(images).local_correlations(swap_dim=False) plt.imshow(Cn, cmap='gray') plt.title('Correlation Image') # plt.show() plt.savefig('figures/correlation_' + dataset_name + '.png') ### set up some parameters is_patches = False # flag for processing in patches or not if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 10 # half size of each patch stride = 4 # overlap between patches K = 3 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = _k # number of neurons expected (in the whole FOV) gSig = [_g, _g] # expected half size of neurons merge_thresh = _merge # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order ### Now RUN CNMF cnm = cnmf.CNMF(n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview, gnb=gnb, rf=rf, stride=stride, rolling_sum=False) cnm = cnm.fit(images) ### plot contour plots of components plt.figure() crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of components') # plt.show() plt.savefig('figures/contour_' + dataset_name + '.png') ### COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) fr = 10 # approximate frame rate of data decay_time = 5.0 # length of transient min_SNR = 2.5 # peak SNR for accepted components (if above this, acept) rval_thr = 0.90 # space correlation threshold (if above this, accept) use_cnn = True # use the CNN classifier min_cnn_thr = 0.10 # if cnn classifier predicts below this value, reject idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=use_cnn, thresh_cnn_lowest=min_cnn_thr) ### visualize selected and rejected components plt.figure() plt.subplot(1, 2, 1) cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=0.9) plt.title('Selected components') plt.savefig('figures/selected_' + dataset_name + '.png') plt.subplot(1, 2, 2) plt.title('Discaded components') cm.utils.visualization.plot_contours(cnm.A[:, idx_components_bad], Cn, thr=0.9) plt.savefig('figures/discaded_' + dataset_name + '.png') # plt.show(block=True) ### visualize selected components cm.utils.visualization.view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components, :], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components, :], img=Cn) ### STOP CLUSTER and clean up log files cm.stop_server() log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file) dview.terminate() return cnm.A.tocsc()[:, idx_components]
def run(batch_dir: str, UUID: str): start_time = time() output = {'status': 0, 'output_info': ''} n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) file_path = batch_dir + '/' + UUID filename = file_path + '.tiff' input_params = pickle.load(open(file_path + '.params', 'rb')) frate = input_params['frate'] gSig = input_params['gSig'] gSiz = 3 * gSig + 1 min_corr = input_params['min_corr'] min_pnr = input_params['min_pnr'] min_SNR = input_params['min_SNR'] r_values_min = input_params['r_values_min'] decay_time = input_params['decay_time'] rf = input_params['rf'] stride = input_params['stride'] gnb = input_params['gnb'] nb_patch = input_params['nb_patch'] k = input_params['k'] if 'Ain' in input_params.keys(): if input_params['Ain']: print('>> Ain specified, looking for cnm-A file <<') item_uuid = input_params['Ain'] parent_batch_dir = os.environ['CURR_BATCH_DIR'] item_out_file = os.path.join(parent_batch_dir, f'{item_uuid}.out') t0 = time() timeout = 60 while not os.path.isfile(item_out_file): print('>>> cnm-A not found, waiting for 15 seconds <<<') sleep(15) if time() - t0 > timeout: output.update({'status': 0, 'output_info': 'Timeout exceeding in waiting for Ain input file'}) raise TimeoutError('Timeout exceeding in waiting for Ain input file') if os.path.isfile(item_out_file): if json.load(open(item_out_file, 'r'))['status']: Ain_file = os.path.join(parent_batch_dir, item_uuid + '_cnm-A.pikl') Ain = pickle.load(open(Ain_file, 'rb')) print('>>> Found Ain file <<<') else: raise FileNotFoundError('>>> Could not find specified Ain file <<<') else: Ain = None else: Ain = None if 'method_deconvolution' in input_params.keys(): method_deconvolution = input_params['method_deconvolution'] else: method_deconvolution = 'oasis' if 'deconv_flag' in input_params.keys(): deconv_flag = input_params['deconv_flag'] else: deconv_flag = True filename = [filename] print('*********** Creating Process Pool ***********') c, dview, n_processes = cm.cluster.setup_cluster(backend='local', # use this one n_processes=n_processes, single_thread=False) if 'bord_px' in input_params.keys(): bord_px = input_params['bord_px'] else: bord_px = 6 try: print('Creating memmap') fname_new = cm.save_memmap_each( filename, base_name='memmap_' + UUID, order='C', border_to_0=bord_px, dview=dview) fname_new = cm.save_memmap_join(fname_new, base_name='memmap_' + UUID, dview=dview) # load memory mappable file Yr, dims, T = cm.load_memmap(fname_new) Y = Yr.T.reshape((T,) + dims, order='F') # compute some summary images (correlation and peak to noise) # change swap dim if output looks weird, it is a problem with tiffile cn_filter, pnr = cm.summary_images.correlation_pnr( Y, gSig=gSig, swap_dim=False) if not input_params['do_cnmfe'] and input_params['do_corr_pnr']: pickle.dump(cn_filter, open(UUID + '_cn_filter.pikl', 'wb'), protocol=4) pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4) output_file_list = [UUID + '_pnr.pikl', UUID + '_cn_filter.pikl', UUID + '_dims.pikl', UUID + '.out' ] output.update({'output': UUID, 'status': 1, 'output_info': 'inspect correlation & pnr', 'output_files': output_file_list }) dview.terminate() for mf in glob(batch_dir + '/memmap_*'): os.remove(mf) end_time = time() processing_time = (end_time - start_time) / 60 output.update({'processing_time': processing_time}) json.dump(output, open(file_path + '.out', 'w')) return cnm = cnmf.CNMF(n_processes=n_processes, method_init='corr_pnr', # use this for 1 photon k=k, # neurons per patch gSig=(gSig, gSig), # half size of neuron gSiz=(gSiz, gSiz), # in general 3*gSig+1 merge_thresh=.3, # threshold for merging p=1, # order of autoregressive process to fit dview=dview, # if None it will run on a single thread # downsampling factor in time for initialization, increase if you have memory problems tsub=2, # downsampling factor in space for initialization, increase if you have memory problems ssub=2, # if you want to initialize with some preselcted components you can pass them here as boolean vectors Ain=Ain, # half size of the patch (final patch will be 100x100) rf=(rf, rf), # overlap among patches (keep it at least large as 4 times the neuron size) stride=(stride, stride), only_init_patch=True, # just leave it as is gnb=gnb, # number of background components nb_patch=nb_patch, # number of background components per patch method_deconvolution=method_deconvolution, # could use 'cvxpy' alternatively deconv_flag=deconv_flag, low_rank_background=True, # leave as is # sometimes setting to False improve the results update_background_components=True, min_corr=min_corr, # min peak value from correlation image min_pnr=min_pnr, # min peak to noise ration from PNR image normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon del_duplicates=True, # whether to remove duplicates from initialization border_pix=bord_px) # number of pixels to not consider in the borders cnm.fit(Y) # DISCARD LOW QUALITY COMPONENTS idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = estimate_components_quality_auto( Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False) # np.save(filename[:-5] + '_curves.npy', cnm.C) pickle.dump(Yr, open(UUID + '_Yr.pikl', 'wb'), protocol=4) pickle.dump(cnm.A, open(UUID + '_cnm-A.pikl', 'wb'), protocol=4) pickle.dump(cnm.b, open(UUID + '_cnm-b.pikl', 'wb'), protocol=4) pickle.dump(cnm.C, open(UUID + '_cnm-C.pikl', 'wb'), protocol=4) pickle.dump(cnm.f, open(UUID + '_cnm-f.pikl', 'wb'), protocol=4) pickle.dump(idx_components, open(UUID + '_idx_components.pikl', 'wb'), protocol=4) pickle.dump(cnm.YrA, open(UUID + '_cnm-YrA.pikl', 'wb'), protocol=4) pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4) pickle.dump(cn_filter, open(UUID + '_cn_filter.pikl', 'wb'), protocol=4) pickle.dump(dims, open(UUID + '_dims.pikl', 'wb'), protocol=4) output_file_list = [UUID + '_cnm-A.pikl', UUID + '_Yr.pikl', UUID + '_cnm-b.pikl', UUID + '_cnm-C.pikl', UUID + '_cnm-f.pikl', UUID + '_idx_components.pikl', UUID + '_cnm-YrA.pikl', UUID + '_pnr.pikl', UUID + '_cn_filter.pikl', UUID + '_dims.pikl', UUID + '.out' ] output.update({'output': filename[:-5], 'status': 1, 'output_files': output_file_list }) except Exception as e: output.update({'status': 0, 'output_info': traceback.format_exc()}) dview.terminate() for mf in glob(batch_dir + '/memmap_*'): os.remove(mf) end_time = time() processing_time = (end_time - start_time) / 60 output.update({'processing_time': processing_time}) json.dump(output, open(file_path + '.out', 'w'))
Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) #%% PLOT COMPONENTS if display_images: plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours(cnm.A[:,
update_background_components=True, # sometimes setting to False improve the results min_corr=min_corr, min_pnr=min_pnr, normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon ssub_B=ssub_B, ring_size_factor=ring_size_factor, del_duplicates=True, # whether to remove duplicates from initialization border_pix=bord_px) # number of pixels to not consider in the borders cnm.fit(Y) #%% DISCARD LOW QUALITY COMPONENTS idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = \ estimate_components_quality_auto( Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False) print(' ***** ') print((len(cnm.C))) print((len(idx_components))) cm.stop_server(dview=dview) #%% PLOT COMPONENTS if display_images: plt.figure(figsize=(12, 6)) plt.subplot(121) crd_good = cm.utils.visualization.plot_contours( cnm.A[:, idx_components], cn_filter, thr=.8, vmax=0.95)
def run_cnmfe(tiff_files, param_file, output_file): """ Run the CNMFe algorithm through CaImAn. :param tiff_files: A list of .tiff files corresponding to a calcium imaging movie. :param param_file: A .yaml parameter file, containing values for the following parameters: num_processes : int The number of processes to run in parallel. The more parallel processes, the more memory that is used. rf : array-like An array [half-width, half-height] that specifies the size of a patch. stride : int The amount of overlap in pixels between patches. K : int The maximum number of cells per patch. gSiz : int The expected diameter of a neuron in pixels. gSig : int The standard deviation a high pass Gaussian filter applied to the movie prior to seed pixel search, roughly equal to the half-size of the neuron in pixels. min_pnr : float The minimum peak-to-noise ratio that is taken into account when searching for seed pixels. min_corr : float The minimum pixel correlation that is taken into account when searching for seed pixels. min_SNR : float Cells with an signal-to-noise (SNR) less than this are rejected. rval_thr : float Cells with a spatial correlation of greater than this are accepted. decay_time : float The expected decay time of a calcium event in seconds. ssub_B : int The spatial downsampling factor used on the background term. merge_threshold : float Cells that are spatially close with a temporal correlation of greater than merge_threshold are automatically merged. :param output_file: The path to a .hdf5 file that will be written to contain the traces, footprints, and deconvolved events identified by CNMFe. """ for tiff_file in tiff_files: if not os.path.exists(tiff_file): raise FileNotFoundError(tiff_file) if not os.path.exists(param_file): raise FileNotFoundError(param_file) with open(param_file, 'r') as f: params = yaml.load(f) expected_params = [ 'gSiz', 'gSig', 'K', 'min_corr', 'min_pnr', 'rf', 'stride', 'decay_time', 'min_SNR', 'rval_thr', 'merge_threshold', 'ssub_B', 'frame_rate', 'num_rows', 'num_cols', 'num_frames', 'num_processes' ] for pname in expected_params: if pname not in params: raise ValueError('Missing parameter {} in file {}'.format( pname, param_file)) gSiz = params['gSiz'] gSig = params['gSig'] K = params['K'] min_corr = params['min_corr'] min_pnr = params['min_pnr'] rf = params['rf'] stride = params['stride'] decay_time = params['decay_time'] min_SNR = params['min_SNR'] rval_thr = params['rval_thr'] merge_threshold = params['merge_threshold'] ssub_B = params['ssub_B'] frame_rate = params['frame_rate'] num_rows = params['num_rows'] num_cols = params['num_cols'] num_frames = params['num_frames'] num_processes = params['num_processes'] # write memmapped file print('Exporting .isxd to memmap file...') mmap_file = _export_movie_to_memmap(tiff_files, num_frames, num_rows, num_cols, overwrite=False) print('Wrote .mmap file to: {}'.format(mmap_file)) # open memmapped file Yr, dims, T = load_memmap(mmap_file) Y = Yr.T.reshape((T, ) + dims, order='F') # grab parallel IPython handle dview = None if num_processes > 1: import ipyparallel as ipp c = ipp.Client() dview = c[:] print('Running using parallel IPython, # clusters = {}'.format( len(c.ids))) num_processes = len(c.ids) # initialize CNMFE parameter object and set user params cnmfe_params = CNMFParams() if gSiz is None: raise ValueError( 'You must set gSiz to an integer, ideally roughly equal to the expected half-cell width.' ) gSiz = _turn_into_array(gSiz) if gSig is None: raise ValueError( 'You must set gSig to a non-zero integer. The default value is 5.') gSig = _turn_into_array(gSig) cnmfe_params.set('preprocess', {'p': 1}) cnmfe_params.set( 'init', { 'K': K, 'min_corr': min_corr, 'min_pnr': min_pnr, 'gSiz': gSiz, 'gSig': gSig }) if rf is None: cnmfe_params.set('patch', {'rf': None, 'stride': 1}) else: cnmfe_params.set('patch', {'rf': np.array(rf), 'stride': stride}) cnmfe_params.set('data', {'decay_time': decay_time}) cnmfe_params.set('quality', {'min_SNR': min_SNR, 'rval_thr': rval_thr}) cnmfe_params.set('merging', {'merge_thr': merge_threshold}) # set parameters that force CNMF into one-photon mode with no temporal or spatial downsampling, # except for the background term cnmfe_params.set( 'init', { 'center_psf': True, 'method_init': 'corr_pnr', 'normalize_init': False, 'nb': -1, 'ssub_B': ssub_B, 'tsub': 1, 'ssub': 1 }) cnmfe_params.set( 'patch', { 'only_init': True, 'low_rank_background': None, 'nb_patch': -1, 'p_tsub': 1, 'p_ssub': 1 }) cnmfe_params.set('spatial', { 'nb': -1, 'update_background_components': False }) cnmfe_params.set('temporal', {'nb': -1, 'p': 1}) # construct and run CNMFE print('Running CNMFe...') cnmfe = CNMF(num_processes, dview=dview, params=cnmfe_params) cnmfe.fit(Y) # run auto accept/reject print('Estimating component quality...') idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = estimate_components_quality_auto( Y, cnmfe.estimates.A, cnmfe.estimates.C, cnmfe.estimates.b, cnmfe.estimates.f, cnmfe.estimates.YrA, frame_rate, cnmfe_params.get('data', 'decay_time'), cnmfe_params.get('init', 'gSiz'), cnmfe.dims, dview=None, min_SNR=cnmfe_params.get('quality', 'min_SNR'), use_cnn=False) save_cnmfe(cnmfe, output_file, good_idx=idx_components)
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # num processes n_proc = 12 # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_proc, single_thread=False) print('Parallel processing initialized.') print('Beginning motion correction') #%%% MOTION CORRECTION t_ms = time.time() # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative if not nomc: mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig,splits_rig=splits_rig,strides=strides, overlaps=overlaps, splits_els=splits_els,upsample_factor_grid=upsample_factor_grid,max_deviation_rigid=max_deviation_rigid,shifts_opencv=True, nonneg_movie=True, use_cuda=use_cuda) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),np.max(np.abs(mc.y_shifts_els)))).astype(np.int) t_mf = time.time() print('Motion correction complete in ', int(t_mf - t_ms),' seconds') # maximum shift to be used for trimming against NaNs #%% compare with original movie #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders bord_px_els = 5 # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_proc, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) print('Beginning initial CNMF fit') t1 = time.time() cnm = cnmf.CNMF(n_processes=n_proc, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) t2 = time.time() print('Initial CNMF fit complete in ', int(t2-t1), 'seconds.') Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier t3 = time.time() print('Estimating component quality') idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) t4 = time.time() print('Component quality estimation complete in ', int(t4-t3),' seconds') #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution print('Re-running CNMF to refine and deconvolve') t5 = time.time() A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=n_proc, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) t6 = time.time() print('CNMF re-run complete in ', int(t6-t5), ' seconds') #%% Extract DF/F values # F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, # quantileMin=8, frames_window=250) save_results = True if save_results: if os.path.isdir(infile[0]): outfile = os.path.join(infile[0],'caiman_output.npz') else: outfile = os.path.join(os.path.split(infile[0])[0],'caiman_output.npz') np.savez_compressed(outfile,Cn=Cn, A=cnm2.A.todense(), C=cnm2.C,b=cnm2.b, f=cnm2.f, YrA=cnm2.YrA, d1=d1, d2=d2,idx_components=idx_components, idx_components_bad=idx_components_bad) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview)
def process_data(haussio_data, mask=None, p=2, nrois_init=400, roi_iceberg=0.9, merge_unconnected=None): if mask is not None: raise RuntimeError("mask not supported in cnmf.process_data") fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' shapefn = os.path.join(haussio_data.dirname_comp, haussio.THOR_RAW_FN[:-3] + "shape.npy") shape = np.load(shapefn) if len(shape) == 5: d1, d2 = shape[2], shape[3] else: d1, d2 = shape[1], shape[2] fn_mmap = get_mmap_name(haussio_data.dirname_comp + os.path.sep + 'Yr', d1, d2, shape[0]) tiffs_to_cnmf(haussio_data) if os.path.exists(fn_cnmf): resdict = loadmat(fn_cnmf) if "dFoF" in resdict.keys(): A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] dFoF = resdict["dFoF"] bl2 = resdict["bl"] f = resdict["f"] images = haussio_data.read_raw().squeeze() else: dFoF = None if not os.path.exists(fn_cnmf) or dFoF is None: c, dview, n_processes = cm.cluster.setup_cluster( backend='multiprocessing', n_processes=NCPUS, single_thread=False) Yr, dims, T = cm.load_memmap(fn_mmap, 'r+') d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') fr = 1.0 / haussio_data.dt # imaging rate in frames per second\n", decay_time = 0.4 # length of a typical transient in seconds\n", # parameters for source extraction and deconvolution\n", bord_px_els = 32 # maximum shift to be used for trimming against NaNs p = 1 # order of the autoregressive system\n", gnb = 2 # number of global background components\n", merge_thresh = 0.8 # merging threshold, max correlation allowed\n", rf = int( np.round(np.sqrt(d1 * d2) / nrois_init) ) # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50\n", if rf < 16: rf = 16 stride_cnmf = 6 # amount of overlap between the patches in pixels\n", npatches = np.round(d1 / (rf * 2) * d2 / (rf * 2)) K = nrois_init / npatches # number of components per patch\n", if K < 2: K = 2 print(rf, npatches, K) gSig = [8, 8] # expected half size of neurons\n", init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')\n", is_dendrites = False # flag for analyzing dendritic data\n", alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF\n", # parameters for component evaluation\n", min_SNR = 2.5 # signal to noise ratio for accepting a component\n", rval_thr = 0.8 # space correlation threshold for accepting a component\n", cnn_thr = 0.8 # threshold for CNN based classifier" cnm = caiman_cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview = dview, min_SNR=min_SNR, r_values_min = rval_thr, use_cnn = False, thresh_cnn_lowest = cnn_thr) A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = caiman_cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) if merge_unconnected is not None: idx_merge = [] for nroi, ca_roi in enumerate(cnm2.C): for nroi_compare_counter, ca_roi_compare in enumerate( cnm2.C[nroi + 1:]): nroi_compare = nroi_compare_counter + nroi + 1 if nroi_compare not in idx_merge: correls = np.correlate(ca_roi, ca_roi_compare, mode='same') correls /= np.sqrt( np.dot(ca_roi, ca_roi) * np.dot(ca_roi_compare, ca_roi_compare)) if correls.max() > merge_unconnected: print("Merging ", nroi_compare) idx_merge.append(nroi_compare) idx_no_merge = [ idx for idx in range(cnm2.C.shape[0]) if idx not in idx_merge ] else: idx_no_merge = range(cnm2.C.shape[0]) A2 = cnm2.A[:, idx_no_merge].tocsc() C2 = cnm2.C[idx_no_merge] YrA = cnm2.YrA[idx_no_merge] S2 = cnm2.S[idx_no_merge] dFoF = cnm2.detrend_df_f(frames_window=300)[idx_no_merge] # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise", i.e. traces = C+YrA) # S: Spikes # f: temporal background savemat( fn_cnmf, { "A": A2, "C": C2, "YrA": YrA, "S": S2, "dFoF": dFoF, "bl": cnm2.b, "f": cnm2.f }) dview.terminate() cm.stop_server() proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(images) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) logfiles = glob.glob("*LOG*") for logfile in logfiles: try: os.unlink(logfile) except OSError: pass polygons = contour(A2, images.shape[1], images.shape[2], thr=roi_iceberg) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, zproj, S2, images, YrA
Cn = cm.local_correlations(images.transpose(1,2,0)) Cn[np.isnan(Cn)] = 0 plt.figure(); crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, SNR_comp_delta, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview = dview, min_SNR=min_SNR, r_values_min = rval_thr, use_cnn = False, thresh_cnn_lowest = cnn_thr) #%% PLOT COMPONENTS plt.figure(); plt.subplot(121); crd_good = cm.utils.visualization.plot_contours(cnm.A[:,idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122); crd_bad = cm.utils.visualization.plot_contours(cnm.A[:,idx_components_bad], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1],YrA=cnm.YrA[idx_components], img=Cn)
def run(batch_dir: str, UUID: str): output = {'status': 0, 'output_info': ''} n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) file_path = batch_dir + '/' + UUID filename = [file_path + '.tiff'] input_params = pickle.load(open(file_path + '.params', 'rb')) fr = input_params['fr'] p = input_params['p'] gnb = input_params['gnb'] merge_thresh = input_params['merge_thresh'] rf = input_params['rf'] stride_cnmf = input_params['stride_cnmf'] K = input_params['k'] gSig = input_params['gSig'] gSig = [gSig, gSig] min_SNR = input_params['min_SNR'] rval_thr = input_params['rval_thr'] cnn_thr = input_params['cnn_thr'] decay_time = input_params['decay_time'] bord_px = input_params['bord_px'] refit = input_params['refit'] print('*********** Creating Process Pool ***********') c, dview, np = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False) try: print('Creating memmap') fname_new = cm.save_memmap_each( filename, base_name='memmap_' + UUID, order='C', border_to_0=bord_px, dview=dview) fname_new = cm.save_memmap_join(fname_new, base_name='memmap_' + UUID, dview=dview) Yr, dims, T = cm.load_memmap(fname_new) Y = Yr.T.reshape((T,) + dims, order='F') cnm = cnmf.CNMF(n_processes=n_processes, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init='greedy_roi', alpha_snmf=None, only_init_patch=False, gnb=gnb, border_pix=bord_px) cnm.fit(Y) idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_lowest=cnn_thr) if refit: A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=n_processes, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(Y) cnmA = cnm2.A cnmb = cnm2.b cnmC = cnm2.C cnm_f = cnm2.f cnmYrA = cnm2.YrA else: cnmA = cnm.A cnmb = cnm.b cnmC = cnm.C cnm_f = cnm.f cnmYrA = cnm.YrA pickle.dump(Yr, open(UUID + '_Yr.pikl', 'wb'), protocol=4) pickle.dump(cnmA, open(UUID + '_cnm-A.pikl', 'wb'), protocol=4) pickle.dump(cnmb, open(UUID + '_cnm-b.pikl', 'wb'), protocol=4) pickle.dump(cnmC, open(UUID + '_cnm-C.pikl', 'wb'), protocol=4) pickle.dump(cnm_f, open(UUID + '_cnm-f.pikl', 'wb'), protocol=4) pickle.dump(idx_components, open(UUID + '_idx_components.pikl', 'wb'), protocol=4) pickle.dump(cnmYrA, open(UUID + '_cnm-YrA.pikl', 'wb'), protocol=4) pickle.dump(dims, open(UUID + '_dims.pikl', 'wb'), protocol=4) output_file_list = [UUID + '_cnm-A.pikl', UUID + '_Yr.pikl', UUID + '_cnm-b.pikl', UUID + '_cnm-C.pikl', UUID + '_cnm-f.pikl', UUID + '_idx_components.pikl', UUID + '_cnm-YrA.pikl', UUID + '_dims.pikl', UUID + '.out' ] output.update({'output': UUID, 'status': 1, 'output_files': output_file_list }) except Exception: output.update({'status': 0, 'output_info': traceback.format_exc()}) dview.terminate() for mf in glob(batch_dir + '/memmap_*'): os.remove(mf) json.dump(output, open(file_path + '.out', 'w'))
min_pnr=min_pnr, # min peak to noise ration from PNR image normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon del_duplicates=True) # whether to remove duplicates from initialization cnm.fit(Y) # %% DISCARD LOW QUALITY COMPONENTS idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality_auto( Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False) print(' ***** ') print((len(cnm.C))) print((len(idx_components))) #%% PLOT ALL COMPONENTS crd = cm.utils.visualization.plot_contours(cnm.A, cn_filter, thr=.8, vmax=0.95) #%% PLOT ONLY GOOD QUALITY COMPONENTS crd = cm.utils.visualization.plot_contours(cnm.A.tocsc()[:, idx_components],