def demo(parallel=False): p = 2 # order of the AR model (in general 1 or 2) if parallel: c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) else: n_processes, dview = 2, None # LOAD MOVIE AND MEMORYMAP fname_new = cm.save_memmap([os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')], base_name='Yr', order = 'C') Yr, dims, T = cm.load_memmap(fname_new) # INIT cnm = cnmf.CNMF(n_processes, method_init='greedy_roi', k=30, gSig=[4, 4], merge_thresh=.8, p=p, dview=dview, Ain=None, method_deconvolution='oasis') # FIT images = np.reshape(Yr.T, [T] + list(dims), order='F') cnm = cnm.fit(images) if parallel: cm.cluster.stop_server(dview=dview) # verifying the spatial components npt.assert_allclose(cnm.A.sum(), 281.1, 1e-2) # verifying the temporal components npt.assert_allclose(cnm.C.sum(), 66271668, 1e-2) try: dview.terminate() except: pass
def demix_and_deconvolve_with_cnmf(scan, num_components=200, AR_order=2, merge_threshold=0.8, num_processes=20, num_pixels_per_process=5000, block_size=10000, num_background_components=4, init_method='greedy_roi', soma_radius=(5, 5), snmf_alpha=None, init_on_patches=False, patch_downsampling_factor=None, percentage_of_patch_overlap=None): """ Extract spike train activity from multi-photon scans using CNMF. Uses constrained non-negative matrix factorization to find neurons/components (locations) and their fluorescence traces (activity) in a timeseries of images, and deconvolves them using an autoregressive model of the calcium impulse response function. See Pnevmatikakis et al., 2016 for details. Default values work alright for somatic images. :param np.array scan: 3-dimensional scan (image_height, image_width, num_frames). :param int num_components: An estimate of neurons/spatial components in the scan. :param int AR_order: Order of the autoregressive process used to model the impulse response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay. :param int merge_threshold: Maximal temporal correlation allowed between activity of overlapping components before merging them. :param int num_processes: Number of processes to run in parallel. None for as many processes as available cores. :param int num_pixels_per_process: Number of pixels that a process handles each iteration. :param int block_size: 'number of pixels to process at the same time for dot product' :param int num_background_components: Number of background components to use. :param string init_method: Initialization method for the components. 'greedy_roi':Look for a gaussian-shaped patch, apply rank-1 NMF, store components, calculate residual scan and repeat for num_components. 'sparse_nmf': Regularized non-negative matrix factorization (as impl. in sklearn) 'local_nmf': ... :param (float, float) soma_radius: Estimated neuron radius (in pixels) in y and x. Used in'greedy_roi' initialization to define the size of the gaussian window. :param int snmf_alpha: Regularization parameter (alpha) for the sparse NMF (if used). :param bool init_on_patches: If True, run the initialization methods on small patches of the scan rather than on the whole image. :param int patch_downsampling_factor: Division to the image dimensions to obtain patch dimensions, e.g., if original size is 256 and factor is 10, patches will be 26x26 :param int percentage_of_patch_overlap: Patches are sampled in a sliding window. This controls how much overlap is between adjacent patches (0 for none, 0.9 for 90%) :returns Location matrix (image_height x image_width x num_components). Inferred location of each component. :returns Activity matrix (num_components x num_frames). Inferred fluorescence traces (spike train convolved with the fitted impulse response function). :returns: Inferred location matrix for background components (image_height x image_width x num_background_components). :returns: Inferred activity matrix for background components (image_height x image_width x num_background_components). :returns: Raw fluorescence traces (num_components x num_frames) obtained from the scan minus activity from background and other components. :returns: Spike matrix (num_components x num_frames). Deconvolved spike activity. :returns: Autoregressive process coefficients (num_components x AR_order) used to model the calcium impulse response of each component: c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ... ..note:: Based on code provided by Andrea Giovanucci. ..note:: The produced number of components is not exactly what you ask for because some components will be merged or deleted. ..warning:: Computation- and memory-intensive for big scans. """ import caiman from caiman.source_extraction.cnmf import cnmf # Save as memory mapped file in F order (that's how caiman wants it) mmap_filename = _save_as_memmap(scan, base_name='/tmp/caiman', order='F').filename # 'Load' scan mmap_scan, (image_height, image_width), num_frames = caiman.load_memmap(mmap_filename) images = np.reshape(mmap_scan.T, (num_frames, image_height, image_width), order='F') # Start the ipyparallel cluster client, direct_view, num_processes = caiman.cluster.setup_cluster( n_processes=num_processes) # Optionally, run the initialization method in small patches to initialize components initial_A = None initial_C = None initial_f = None if init_on_patches: # Calculate patch size (only square patches allowed) bigger_dimension = max(image_height, image_width) smaller_dimension = min(image_height, image_width) patch_size = bigger_dimension / patch_downsampling_factor patch_size = min(patch_size, smaller_dimension) # if bigger than small dimension # Calculate num_components_per_patch num_nonoverlapping_patches = (image_height/patch_size) * (image_width/patch_size) num_components_per_patch = num_components / num_nonoverlapping_patches num_components_per_patch = max(num_components_per_patch, 1) # at least 1 # Calculate patch overlap in pixels overlap_in_pixels = patch_size * percentage_of_patch_overlap # Make sure they are integers patch_size = int(round(patch_size)) num_components_per_patch = int(round(num_components_per_patch)) overlap_in_pixels = int(round(overlap_in_pixels)) # Run CNMF on patches (only for initialization, no impulse response modelling p=0) model = cnmf.CNMF(num_processes, only_init_patch=True, p=0, rf=int(round(patch_size / 2)), stride=overlap_in_pixels, k=num_components_per_patch, merge_thresh=merge_threshold, method_init=init_method, gSig=soma_radius, alpha_snmf=snmf_alpha, gnb=num_background_components, n_pixels_per_process=num_pixels_per_process, block_size=block_size, check_nan=False, dview=direct_view, method_deconvolution='cvxpy') model = model.fit(images) # Delete log files (one per patch) log_files = glob.glob('caiman*_LOG_*') for log_file in log_files: os.remove(log_file) # Get results initial_A = model.A initial_C = model.C initial_f = model.f # Run CNMF model = cnmf.CNMF(num_processes, k=num_components, p=AR_order, merge_thresh=merge_threshold, gnb=num_background_components, method_init=init_method, gSig=soma_radius, alpha_snmf=snmf_alpha, n_pixels_per_process=num_pixels_per_process, block_size=block_size, check_nan=False, dview=direct_view, Ain=initial_A, Cin=initial_C, f_in=initial_f, method_deconvolution='cvxpy') model = model.fit(images) # Get final results location_matrix = model.A # pixels x num_components activity_matrix = model.C # num_components x num_frames background_location_matrix = model.b # pixels x num_background_components background_activity_matrix = model.f # num_background_components x num_frames spikes = model.S # num_components x num_frames, spike_ traces raw_traces = model.C + model.YrA # num_components x num_frames AR_coefficients = model.g # AR_order x num_components # Reshape spatial matrices to be image_height x image_width x num_frames new_shape = (image_height, image_width, -1) location_matrix = location_matrix.toarray().reshape(new_shape, order='F') background_location_matrix = background_location_matrix.reshape(new_shape, order='F') AR_coefficients = np.array(list(AR_coefficients)) # unwrapping it (num_components x 2) # Stop ipyparallel cluster client.close() caiman.stop_server() # Delete memory mapped scan os.remove(mmap_filename) return (location_matrix, activity_matrix, background_location_matrix, background_activity_matrix, raw_traces, spikes, AR_coefficients)
from caiman.source_extraction.cnmf.estimates import Estimates import scipy estimates = vpy.estimates.copy() A = np.array(estimates['spatial_filter']).transpose([1,2,0]).reshape((-1, len(estimates['spatial_filter'])),order='F') A = A / A.max(axis=0) b = np.zeros((A.shape[0],2)) A = scipy.sparse.csc_matrix(A) b = scipy.sparse.csc_matrix(b) C = np.array(estimates['t_rec']) f = np.zeros((2, C.shape[1])) R = np.array(estimates['t']) - C est = Estimates(A=A, C=C, b=b, f=f, R=R, dims=(100,100)) est.YrA = R #est.plot_contours(img=summary_image[:,:,2]) # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') est.dview = dview #est.view_components(img=summary_image[:,:,2]) est.play_movie(imgs=images, magnification=4) est = np.load('/home/nel/data/voltage_data/volpy_paper/reconstructed/estimates.npz',allow_pickle=True)['arr_0'].item() fnames = ['/home/nel/data/voltage_data/volpy_paper/memory/403106_3min_10000._rig__d1_512_d2_128_d3_1_order_F_frames_10000_.mmap']
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between patches (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = False if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie, m_els.resize(1, 1, ds_ratio)], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = {'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub} opts.change_params(params_dict=opts_dict) # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
def test_general(): """ General Test of pipeline with comparison against ground truth A shorter version than the demo pipeline that calls comparison for the real test work Raises: --------- params_movie params_cnmf rig correction cnmf on patch cnmf full frame not able to read the file no groundtruth """ #\bug #\warning global params_movie global params_diplay fname = params_movie['fname'] niter_rig = params_movie['niter_rig'] max_shifts = params_movie['max_shifts'] splits_rig = params_movie['splits_rig'] num_splits_to_process_rig = params_movie['num_splits_to_process_rig'] cwd = os.getcwd() fname = download_demo(fname[0]) m_orig = cm.load(fname) min_mov = m_orig[:400].min() comp = comparison.Comparison() comp.dims = np.shape(m_orig)[1:] ################ RIG CORRECTION ################# t1 = time.time() mc = MotionCorrect(fname, min_mov, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, num_splits_to_process_rig=num_splits_to_process_rig, shifts_opencv=True, nonneg_movie=True) mc.motion_correct_rigid(save_movie=True) m_rig = cm.load(mc.fname_tot_rig) bord_px_rig = np.ceil(np.max(mc.shifts_rig)).astype(np.int) comp.comparison['rig_shifts']['timer'] = time.time() - t1 comp.comparison['rig_shifts']['ourdata'] = mc.shifts_rig ########################################### if 'max_shifts' not in params_movie: fnames = params_movie['fname'] border_to_0 = 0 else: # elif not params_movie.has_key('overlaps'): fnames = mc.fname_tot_rig border_to_0 = bord_px_rig m_els = m_rig idx_xy = None add_to_movie = -np.nanmin(m_els) + 1 # movie must be positive remove_init = 0 downsample_factor = 1 base_name = fname[0].split('/')[-1][:-4] name_new = cm.save_memmap_each(fnames, base_name=base_name, resize_fact=(1, 1, downsample_factor), remove_init=remove_init, idx_xy=idx_xy, add_to_movie=add_to_movie, border_to_0=border_to_0) name_new.sort() if len(name_new) > 1: fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=params_movie['n_chunks'], dview=None) else: logging.warning('One file only, not saving!') fname_new = name_new[0] Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') if np.min(images) < 0: # TODO: should do this in an automatic fashion with a while loop at the 367 line raise Exception('Movie too negative, add_to_movie should be larger') if np.sum(np.isnan(images)) > 0: # TODO: same here raise Exception( 'Movie contains nan! You did not remove enough borders') Cn = cm.local_correlations(Y) Cn[np.isnan(Cn)] = 0 p = params_movie['p'] merge_thresh = params_movie['merge_thresh'] rf = params_movie['rf'] stride_cnmf = params_movie['stride_cnmf'] K = params_movie['K'] init_method = params_movie['init_method'] gSig = params_movie['gSig'] if params_movie['is_dendrites'] == True: if params_movie['init_method'] is not 'sparse_nmf': raise Exception('dendritic requires sparse_nmf') if params_movie['alpha_snmf'] is None: raise Exception('need to set a value for alpha_snmf') ################ CNMF PART PATCH ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], dview=None, rf=rf, stride=stride_cnmf, memory_fact=params_movie['memory_fact'], method_init=init_method, alpha_snmf=100, only_init_patch=params_movie['only_init_patch'], gnb=params_movie['gnb'], method_deconvolution='oasis') comp.cnmpatch = copy.copy(cnm) comp.cnmpatch.estimates = None cnm = cnm.fit(images) A_tot = cnm.estimates.A C_tot = cnm.estimates.C YrA_tot = cnm.estimates.YrA b_tot = cnm.estimates.b f_tot = cnm.estimates.f # DISCARDING logging.info(('Number of components:' + str(A_tot.shape[-1]))) final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_patch'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_patch'] fitness_delta_min = params_movie['fitness_delta_min_patch'] Npeaks = params_movie['Npeaks'] traces = C_tot + YrA_tot idx_components, idx_components_bad = estimate_components_quality( traces, Y, A_tot, C_tot, b_tot, f_tot, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min) ####### A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1 comp.comparison['cnmf_on_patch']['ourdata'] = [A_tot.copy(), C_tot.copy()] #################### ######################## ################ CNMF PART FULL ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, p=p, Ain=A_tot, Cin=C_tot, f_in=f_tot, rf=None, stride=None, method_deconvolution='oasis') cnm = cnm.fit(images) # DISCARDING A, C, b, f, YrA, sn = cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, cnm.estimates.sn final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_full'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_full'] fitness_delta_min = params_movie['fitness_delta_min_full'] Npeaks = params_movie['Npeaks'] traces = C + YrA idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality( traces, Y, A, C, b, f, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True) ########## A_tot_full = A_tot.tocsc()[:, idx_components] C_tot_full = C_tot[idx_components] comp.comparison['cnmf_full_frame']['timer'] = time.time() - t1 comp.comparison['cnmf_full_frame']['ourdata'] = [ A_tot_full.copy(), C_tot_full.copy() ] #################### ######################## comp.save_with_compare(istruth=False, params=params_movie, Cn=Cn) log_files = glob.glob('*_LOG_*') try: for log_file in log_files: os.remove(log_file) except: logging.warning('Cannot remove log files') ############ assertions ################## pb = False if (comp.information['differences']['params_movie']): logging.error( "you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" ) pb = True if (comp.information['differences']['params_cnm']): logging.warning( "you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" ) # pb = True if (comp.information['diff']['rig']['isdifferent']): logging.error("the rigid shifts are different from the groundtruth ") pb = True if (comp.information['diff']['cnmpatch']['isdifferent']): logging.error( "the cnmf on patch produces different results than the groundtruth " ) pb = True if (comp.information['diff']['cnmfull']['isdifferent']): logging.error( "the cnmf full frame produces different results than the groundtruth " ) pb = True assert (not pb)
def substract_neuropil(data_path, agonia_th, neuropil_pctl, signal_pctl): '''Calculate the neuropil trace as the average fluorescence of every pixel that is not in a box. subtract it to every box trace, calculated as the activity of the pixels above the signal_pctl. Parameters ---------- data_path : string path to folder containing the data agonia_th : float threshold for detection confidence for each box neuropil_pctl : int percentile of pixels intensity bellow which pixel is considered to calculate neuropil signal_pctl : int percentile of pixels intensity above which pixel is considered to calculate trace Returns ------- denoised_traces : NxT ndarray temporal traces after neuropil subtraction for the N boxes neuropil_trace : array normalized temporal trace of background noise, length T neuropil_power : float mean of neuropil trace before normalization ''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load Caiman results cnm = cnmf.load_CNMF(results_caiman_path) # calculate the centers of the CaImAn factors centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() # keep only cells above confidence threshold boxes = boxes[boxes[:, 4] > agonia_th].astype('int') #delete boxes that do not have a caiman cell inside k = 0 for cell, box in enumerate(boxes): idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] if not idx_factor: boxes = np.delete(boxes, cell - k, axis=0) k += 1 boxes_traces = np.empty((boxes.shape[0], images.shape[0])) mask = np.zeros(images.shape[1:], dtype=bool) for cell, box in enumerate(boxes): #boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2)) med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0) box_trace = images[:, box[1]:box[3], box[0]:box[2]] boxes_traces[cell] = box_trace[:, med > np.percentile(med, signal_pctl )].mean(axis=1) mask[box[1].astype('int'):box[3].astype('int'), box[0].astype('int'):box[2].astype('int')] = 1 mask = (1 - mask).astype('bool') not_cell = images[:, mask] #not_cell_med = np.median(not_cell,axis=0) #neuropil_trace = not_cell[:,not_cell_med<np.percentile(not_cell_med,neuropil_pctl)].mean(axis=1) #denoised_traces = boxes_traces-neuropil_trace neuropil_trace = not_cell.mean(axis=1) neuropil_power = neuropil_trace.mean() neuropil_trace = neuropil_trace / neuropil_power denoised_traces = np.array([ boxes_traces[i] - neuropil_trace * boxes_traces[i].mean() for i in range(boxes_traces.shape[0]) ]) return denoised_traces, neuropil_trace, neuropil_power
def trace_correlation(data_path, agonia_th, select_cells=False, plot_results=True, denoise=False): '''Calculate the correlation between the mean of the Agonia Box and the CaImAn factor. Parameters ---------- data_path : string path to folder containing the data agonia_th : float threshold for detection confidence for each box select_cells: bool, optional if True get index of active cells plot_results: bool, optional if True do boxplot of correlation values for all cells, if selected_cells, use only active cells denoise : bool, optional if True subtract neuropil Returns ------- cell_corr : numpy array corrcoef value for all cells idx_active : list if select_cells=True returns index of active cells, otherwise returns index of all cells boxes_traces : NxT ndarray temporal trace for the N box that has an asociated caiman factor''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) # load Caiman results cnm = cnmf.load_CNMF(results_caiman_path) # calculate the centers of the CaImAn factors centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) # load boxes with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() # keep only cells above confidence threshold boxes = boxes[boxes[:, 4] > agonia_th].astype('int') #delete boxes that do not have a caiman cell inside k = 0 for cell, box in enumerate(boxes): idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] if not idx_factor: boxes = np.delete(boxes, cell - k, axis=0) k += 1 # Load video as 3D tensor (each plane is a frame) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') boxes_traces = np.empty((boxes.shape[0], images.shape[0])) #calculate correlations between boxes and CaImAn factors cell_corr = np.empty(len(boxes_traces)) neuropil_trace = np.zeros(T) if denoise: _, neuropil_trace, neuropil_power = substract_neuropil( data_path, agonia_th, 100, 80) for cell, box in enumerate(boxes): # calculate boxes traces as means over images #boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2))-neuropil_trace boxes_traces[cell] = images[:, box[1]:box[3], box[0]:box[2]].mean(axis=(1, 2)) #for using the percentile criteria med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0) box_trace = images[:, box[1]:box[3], box[0]:box[2]] boxes_traces[cell] = box_trace[:, np.logical_and( med > np.percentile(med, 80), med < np.percentile(med, 95))].mean( axis=1) boxes_traces[ cell] = boxes_traces[cell] - neuropil_trace * neuropil_power * .7 #boxes_traces[cell] = boxes_traces[cell]-neuropil_trace*boxes_traces[cell].mean() # get the asociated CaImAn factor by checking if its center of mass is inside the box idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] # in case there is more than one center inside the box choose the one closer to the center of the box if len(idx_factor) > 1: idx_factor = [ idx_factor[np.argmin([ np.linalg.norm([(box[3] - box[1]) / 2, (box[2] - box[0]) / 2] - c) for c in centers[idx_factor] ])] ] cell_corr[cell] = np.corrcoef( [cnm.estimates.C[idx_factor[0]], boxes_traces[cell]])[1, 0] if select_cells: #select only active cells using CaImAn criteria fitness, _, _, _ = compute_event_exceptionality(boxes_traces) idx_active = [cell for cell, fit in enumerate(fitness) if fit < -20] else: idx_active = [cell for cell, _ in enumerate(boxes_traces)] if plot_results: corr_toplot = [ corr for id, corr in enumerate(cell_corr) if id in idx_active and ~np.isnan(corr) ] fig, ax = plt.subplots() p = ax.boxplot(corr_toplot) ax.set_ylim([-1, 1]) plt.text(1.2, 0.8, 'n_cells = {}'.format(len(corr_toplot))) return cell_corr, idx_active, boxes_traces
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set to true to show movies and images fnames = ['data_endoscope.tif'] # filename to be processed frate = 10 # movie frame rate decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters do_motion_correction_nonrigid = True do_motion_correction_rigid = False # in this case it will also save a rigid motion corrected movie gSig_filt = (3, 3) # size of filter, in general gSig (see below), # change this one if algorithm does not work max_shifts = (5, 5) # maximum allowed rigid shift splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time strides = (48, 48) # start a new patch for pw-rigid motion correction every x pixels overlaps = (24, 24) # overlap between pathes (size of patch strides+overlaps) # for parallelization split the movies in num_splits chuncks across time # (remember that it should hold that length_movie/num_splits_to_process_rig>100) splits_els = 10 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None gSig = 3 # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = 13 # average diameter of a neuron, in general 4*gSig+1 merge_thresh = .7 # merging threshold, max correlation allowed rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80 stride_cnmf = 20 # amount of overlap between the patches in pixels # (keep it at least large as gSiz, i.e 4 times the neuron size gSig) tsub = 2 # downsampling factor in time for initialization, # increase if you have memory problems ssub = 1 # downsampling factor in space for initialization, # increase if you have memory problems Ain = None # if you want to initialize with some preselected components # you can pass them here as boolean vectors low_rank_background = None # None leaves background of each patch intact, # True performs global low-rank approximation gnb = -1 # number of background components (rank) if positive, # else exact ring model with following settings # gnb=-2: Return background as b and W # gnb=-1: Return full rank background B # gnb= 0: Don't return background nb_patch = -1 # number of background components (rank) per patch, # use 0 or -1 for exact background of ring model (cf. gnb) min_corr = .8 # min peak value from correlation image min_pnr = 10 # min peak to noise ration from PNR image ssub_B = 2 # additional downsampling factor in space for background ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor # parameters for component evaluation min_SNR = 3 # adaptive way to set threshold on the transient size r_values_min = 0.85 # threshold on space consistency (if you lower more components # will be accepted, potentially with worst quality) #%% start the cluster try: cm.stop_server(dview=dview) # stop it if it was running except: pass c, dview, n_processes = cm.cluster.setup_cluster(backend='local', # use this one n_processes=24, # number of process to use, if you go out of memory try to reduce this one single_thread=False) #%% download demo file fnames = [download_demo(fnames[0])] filename_reorder = fnames #%% MOTION CORRECTION if do_motion_correction_nonrigid or do_motion_correction_rigid: # do motion correction rigid mc = motion_correct_oneP_rigid(fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, dview=dview, splits_rig=splits_rig, save_movie=not(do_motion_correction_nonrigid) ) new_templ = mc.total_template_rig plt.subplot(1, 2, 1) plt.imshow(new_templ) # % plot template plt.subplot(1, 2, 2) plt.plot(mc.shifts_rig) # % plot rigid shifts plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames') plt.ylabel('pixels') # borders to eliminate from movie because of motion correction bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) filename_reorder = mc.fname_tot_rig # do motion correction nonrigid if do_motion_correction_nonrigid: mc = motion_correct_oneP_nonrigid( fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, dview=dview, splits_rig=None, save_movie=True, # whether to save movie in memory mapped format new_templ=new_templ # template to initialize motion correction ) filename_reorder = mc.fname_tot_els bord_px = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # create memory mappable file in the right order on the hard drive (C order) fname_new = cm.save_memmap_each( filename_reorder, base_name='memmap_', order='C', border_to_0=bord_px, dview=dview) fname_new = cm.save_memmap_join(fname_new, base_name='memmap_', dview=dview) # load memory mappable file Yr, dims, T = cm.load_memmap(fname_new) Y = Yr.T.reshape((T,) + dims, order='F') #%% compute some summary images (correlation and peak to noise) # change swap dim if output looks weird, it is a problem with tiffile cn_filter, pnr = cm.summary_images.correlation_pnr(Y, gSig=gSig, swap_dim=False) # inspect the summary images and set the parameters inspect_correlation_pnr(cn_filter, pnr) # print parameters set above, modify them if necessary based on summary images print(min_corr) # min correlation of peak (from correlation image) print(min_pnr) # min peak to noise ratio #%% RUN CNMF ON PATCHES cnm = cnmf.CNMF( n_processes=n_processes, method_init='corr_pnr', # use this for 1 photon k=K, gSig=(gSig, gSig), gSiz=(gSiz, gSiz), merge_thresh=merge_thresh, p=p, dview=dview, tsub=tsub, ssub=ssub, Ain=Ain, rf=rf, stride=stride_cnmf, only_init_patch=True, # just leave it as is gnb=gnb, nb_patch=nb_patch, method_deconvolution='oasis', # could use 'cvxpy' alternatively low_rank_background=low_rank_background, update_background_components=True, # sometimes setting to False improve the results min_corr=min_corr, min_pnr=min_pnr, normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon ssub_B=ssub_B, ring_size_factor=ring_size_factor, del_duplicates=True, # whether to remove duplicates from initialization border_pix=bord_px) # number of pixels to not consider in the borders cnm.fit(Y) #%% DISCARD LOW QUALITY COMPONENTS idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = \ estimate_components_quality_auto( Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False) print(' ***** ') print((len(cnm.C))) print((len(idx_components))) cm.stop_server(dview=dview) #%% PLOT COMPONENTS if display_images: plt.figure(figsize=(12, 6)) plt.subplot(121) crd_good = cm.utils.visualization.plot_contours( cnm.A[:, idx_components], cn_filter, thr=.8, vmax=0.95) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], cn_filter, thr=.8, vmax=0.95) plt.title('Contour plots of rejected components') #%% VISUALIZE IN DETAILS COMPONENTS cm.utils.visualization.view_patches_bar( Yr, cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=cn_filter) #%% MOVIES if display_images: B = cnm.b.dot(cnm.f) if 'sparse' in str(type(B)): B = B.toarray() # denoised movie cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot(cnm.C[idx_components]) + B, dims + (-1,), order='F').transpose(2, 0, 1)).play(magnification=3, gain=1.) # only neurons cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot( cnm.C[idx_components]), dims + (-1,), order='F').transpose(2, 0, 1) ).play(magnification=3, gain=10.) # only the background cm.movie(np.reshape(B, dims + (-1,), order='F').transpose(2, 0, 1) ).play(magnification=3, gain=1.) # residuals cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B, dims + (-1,), order='F').transpose(2, 0, 1) ).play(magnification=3, gain=10., fr=10) # eventually, you can rerun the algorithm on the residuals plt.imshow(cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B, dims + (-1,), order='F').transpose(2, 0, 1) ).local_correlations(swap_dim=False))
def main(): pass # For compatibility between running under Spyder and the CLI # %% start the cluster try: cm.stop_server() # stop it if it was running except (): pass c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes= 24, # number of process to use, if you go out of memory try to reduce this one single_thread=False) # %% First setup some parameters for motion correction # dataset dependent parameters fnames = ['data_endoscope.tif'] # filename to be processed fnames = [download_demo(fnames[0])] # download file if not already present filename_reorder = fnames fr = 10 # movie frame rate decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters motion_correct = True # flag for motion correction pw_rigid = False # flag for pw-rigid motion correction gSig_filt = (3, 3) # size of filter, in general gSig (see below), # change this one if algorithm does not work max_shifts = (5, 5) # maximum allowed rigid shift strides = ( 48, 48 ) # start a new patch for pw-rigid motion correction every x pixels overlaps = (24, 24 ) # overlap between pathes (size of patch strides+overlaps) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 border_nan = 'copy' mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'gSig_filt': gSig_filt, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': border_nan } opts = params.CNMFParams(params_dict=mc_dict) # %% MOTION CORRECTION # The pw_rigid flag set above, determines where to use rigid or pw-rigid # motion correction if motion_correct: # do motion correction rigid mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) mc.motion_correct(save_movie=True) fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig if pw_rigid: bord_px = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) else: # marker if show_chart: bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) plt.subplot(1, 2, 1) plt.imshow(mc.total_template_rig) # % plot template plt.subplot(1, 2, 2) plt.plot(mc.shifts_rig) # % plot rigid shifts plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames') plt.ylabel('pixels') bord_px = 0 if border_nan is 'copy' else bord_px fname_new = cm.save_memmap(fname_mc, base_name='memmap_', order='C', border_to_0=bord_px) else: # if no motion correction just memory map the file fname_new = cm.save_memmap(filename_reorder, base_name='memmap_', order='C', border_to_0=0, dview=dview) # load memory mappable file Yr, dims, T = cm.load_memmap(fname_new) images = Yr.T.reshape((T, ) + dims, order='F') # %% Parameters for source extraction and deconvolution (CNMF-E algorithm) p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None for 1p data gSig = ( 3, 3 ) # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1 Ain = None # possibility to seed with predetermined binary masks merge_thr = .7 # merging threshold, max correlation allowed rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80 stride_cnmf = 20 # amount of overlap between the patches in pixels # (keep it at least large as gSiz, i.e 4 times the neuron size gSig) tsub = 2 # downsampling factor in time for initialization, # increase if you have memory problems ssub = 1 # downsampling factor in space for initialization, # increase if you have memory problems # you can pass them here as boolean vectors low_rank_background = None # None leaves background of each patch intact, # True performs global low-rank approximation if gnb>0 gnb = 0 # number of background components (rank) if positive, # else exact ring model with following settings # gnb= 0: Return background as b and W # gnb=-1: Return full rank background B # gnb<-1: Don't return background nb_patch = 0 # number of background components (rank) per patch if gnb>0, # else it is set automatically min_corr = .8 # min peak value from correlation image min_pnr = 10 # min peak to noise ration from PNR image ssub_B = 2 # additional downsampling factor in space for background ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor opts.change_params( params_dict={ 'dims': dims, 'method_init': 'corr_pnr', # use this for 1 photon 'K': K, 'gSig': gSig, 'gSiz': gSiz, 'merge_thr': merge_thr, 'p': p, 'tsub': tsub, 'ssub': ssub, 'rf': rf, 'stride': stride_cnmf, 'only_init': True, # set it to True to run CNMF-E 'nb': gnb, 'nb_patch': nb_patch, 'method_deconvolution': 'oasis', # could use 'cvxpy' alternatively 'low_rank_background': low_rank_background, 'update_background_components': True, # sometimes setting to False improve the results 'min_corr': min_corr, 'min_pnr': min_pnr, 'normalize_init': False, # just leave as is 'center_psf': True, # leave as is for 1 photon 'ssub_B': ssub_B, 'ring_size_factor': ring_size_factor, 'del_duplicates': True, # whether to remove duplicates from initialization 'border_pix': bord_px }) # number of pixels to not consider in the borders) # %% compute some summary images (correlation and peak to noise) # change swap dim if output looks weird, it is a problem with tiffile cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig=gSig[0], swap_dim=False) # if your images file is too long this computation will take unnecessarily # long time and consume a lot of memory. Consider changing images[::1] to # images[::5] or something similar to compute on a subset of the data # inspect the summary images and set the parameters inspect_correlation_pnr(cn_filter, pnr) # print parameters set above, modify them if necessary based on summary images print(min_corr) # min correlation of peak (from correlation image) print(min_pnr) # min peak to noise ratio # %% RUN CNMF ON PATCHES cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts) cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% DISCARD LOW QUALITY COMPONENTS min_SNR = 2.5 # adaptive way to set threshold on the transient size r_values_min = 0.85 # threshold on space consistency (if you lower more components # will be accepted, potentially with worst quality) cnm.params.set('quality', { 'min_SNR': min_SNR, 'rval_thr': r_values_min, 'use_cnn': False }) cnm.estimates.evaluate_components(images, cnm.params, dview=dview) print(' ***** ') print('Number of total components: ', len(cnm.estimates.C)) print('Number of accepted components: ', len(cnm.estimates.idx_components)) # %% PLOT COMPONENTS cnm.dims = dims display_images = False # Set to true to show movies and images if display_images: cnm.estimates.plot_contours(img=cn_filter, idx=cnm.estimates.idx_components) cnm.estimates.view_components(images, idx=cnm.estimates.idx_components) # %% MOVIES display_images = False # Set to true to show movies and images if display_images: # fully reconstructed movie cnm.estimates.play_movie(images, q_max=99.5, magnification=2, include_bck=True, gain_res=10, bpx=bord_px) # movie without background cnm.estimates.play_movie(images, q_max=99.9, magnification=2, include_bck=False, gain_res=4, bpx=bord_px) # %% STOP SERVER cm.stop_server(dview=dview)
def run_CaImAn_session(pathSession, suffix="", use_parallel=True): pass # For compatibility between running under Spyder and the CLI ### %% set paths #pathSession = "/media/wollex/Analyze_AS3/Data/879/Session01/" #pathSession = "/home/wollex/Data/Documents/Uni/2016-XXXX_PhD/Japan/Work/Data/M879/Session01" #sv_dir = "/home/wollex/Data/Documents/Uni/2016-XXXX_PhD/Japan/Work/Data/tmp/" sv_dir = "/home/aschmidt/Documents/Data/tmp/" fname = None for f in os.listdir(pathSession): if f.startswith("thy") or f.startswith("shank"): fname = pathSession + f if f.endswith('.h5'): break if not fname or not os.path.exists(fname): print("No file here to process :(") return svname = pathSession + "results_OnACID.mat" #if os.path.exists(svname): #print("Processed file already present - skipping") #return svname_h5 = pathSession + "results_OnACID.hdf5" t_start = time.time() border_thr = 5 # minimal distance of centroid to border # set up CNMF parameters params_dict = { #general data 'fnames': [fname], 'fr': 15, 'decay_time': 0.47, 'gSig': [6, 6], # expected half size of neurons #model/analysis 'rf': 64 // 2, # size of patch 'K': 200, # max number of components 'nb': 2, # number of background components per patch 'p': 0, # order of AR indicator dynamics 'stride': 8, #'simultaneously': True, # init 'ssub': 2, # spatial subsampling during initialization 'tsub': 5, # temporal subsampling during initialization #motion 'motion_correct': False, 'pw_rigid': False, 'strides': 96, 'max_shifts': 12, # maximum allowed rigid shift in pixels 'overlaps': 24, # overlap between patches (size of patch in pixels: strides+overlaps) 'max_deviation_rigid': 12, # maximum deviation allowed for patch with respect to rigid shifts #'only_init': False, # whether to run only the initialization #online 'init_batch': 300, # number of frames for initialization 'init_method': 'bare', # initialization method 'update_freq': 2000, # update every shape at least once every update_freq steps 'use_dense': False, #'dist_shape_update': True, #make things more memory efficient 'memory_efficient': False, 'block_size_temp': 5000, 'num_blocks_per_run_temp': 20, 'block_size_spat': 5000, 'num_blocks_per_run_spat': 20, #quality 'min_SNR': 2.5, # minimum SNR for accepting candidate components 'rval_thr': 0.85, # space correlation threshold for accepting a component 'rval_lowest': 0, 'sniper_mode': True, # flag for using CNN for detecting new components #'test_both': True, # use CNN and correlation to test for new components 'use_cnn': True, 'thresh_CNN_noisy': 0.6, # CNN threshold for candidate components 'min_cnn_thr': 0.8, # threshold for CNN based classifier 'cnn_lowest': 0.3, # neurons with cnn probability lower than this value are rejected #display 'show_movie': False, 'save_online_movie': False, 'movie_name_online': "test_mp4v.avi" } opts = cnmf.params.CNMFParams(params_dict=params_dict) print("Start writing memmapped file @t = " + time.ctime()) if use_parallel: c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) else: dview = None n_processes = 1 fname_memmap = cm.save_memmap([fname], base_name='memmap%s_' % suffix, save_dir=sv_dir, n_chunks=20, order='C', dview=dview) # exclude borders if use_parallel: cm.stop_server(dview=dview) ## restart server to clean up memory print(fname_memmap) if not fname.endswith('.h5'): print('perform motion correction') # first we create a motion correction object with the specified parameters #mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion')) #%% Run (piecewise-rigid motion) correction using NoRMCorre #mc.motion_correct(save_movie=True) #fname_memmap = cm.save_memmap(mc.mmap_file, base_name='memmap_', save_dir=sv_dir, n_chunks=20, order='C', dview=dview) # exclude borders #os.remove(mc.mmap_file[0]) opts.change_params({'motion_correct': True, 'pw_rigid': True}) print("Done @t = %s, (time passed: %s)" % (time.ctime(), str(time.time() - t_start))) #fname_memmap = sv_dir + "memmap__d1_512_d2_512_d3_1_order_C_frames_8989_.mmap" opts.change_params({'fnames': [fname_memmap]}) Cn = cm.load(fname_memmap, subindices=slice(0, None, 5)).local_correlations(swap_dim=False) Yr, dims, T = cm.load_memmap(fname_memmap) Y = np.reshape(Yr.T, [T] + list(dims), order='F') ### ------------------ 1st run ------------------ ### ### %% fit with online object on memmapped data cnm = cnmf.online_cnmf.OnACID(params=opts) cnm.fit_online() print('Number of components found:' + str(cnm.estimates.A.shape[-1])) plt.close('all') ### %% evaluate components (CNN, SNR, correlation, border-proximity) if use_parallel: c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) else: dview = None n_processes = 1 cnm.estimates.evaluate_components(Y, opts, dview) #cnm.estimates.view_components(img=Cn) #plt.close('all') cnm.estimates.plot_contours( img=Cn, idx=cnm.estimates.idx_components, crd=None) ## plot contours, need that one to get the coordinates plt.draw() plt.pause(1) idx_border = [] for n in cnm.estimates.idx_components: if (cnm.estimates.coordinates[n]['CoM'] < border_thr).any() or ( cnm.estimates.coordinates[n]['CoM'] > (cnm.estimates.dims[0] - border_thr)).any(): idx_border.append(n) cnm.estimates.idx_components = np.setdiff1d(cnm.estimates.idx_components, idx_border) cnm.estimates.idx_components_bad = np.union1d( cnm.estimates.idx_components_bad, idx_border) cnm.estimates.select_components( use_object=True, save_discarded_components=False ) #%% update object with selected components print('Number of components left after evaluation:' + str(cnm.estimates.A.shape[-1])) ### %% save file to allow loading as CNMF- (instead of OnACID-) file #cnm.estimates = clear_cnm(cnm.estimates,remove=['shifts','discarded_components']) cnm.estimates = clear_cnm( cnm.estimates, retain=['A', 'C', 'S', 'b', 'f', 'YrA', 'dims', 'coordinates', 'sn']) cnm.save(svname_h5) ### -------------------2nd run --------------------- ### ### %% run a refit on the whole data if use_parallel: cm.stop_server(dview=dview) ## restart server to clean up memory c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) else: dview = None n_processes = 1 cnm = cnmf.cnmf.load_CNMF(svname_h5, n_processes, dview) cnm.params.change_params({'p': 1}) cnm.estimates.dims = Cn.shape # gets lost for some reason print( "merge & update spatial + temporal & deconvolve @t = %s, (time passed: %s)" % (time.ctime(), str(time.time() - t_start))) cnm.update_temporal( Yr) # need this to calculate noise per component for merging purposes cnm.merge_comps(Yr, mx=1000, fast_merge=False) cnm.estimates.C[np.where( np.isnan(cnm.estimates.C) )] = 0 ## for some reason, there are NaNs in it -> cant process this cnm.update_temporal(Yr) # update temporal trace after merging cnm.params.change_params({'n_pixels_per_process': 1000}) ## for some reason this one gets lost cnm.estimates.sn, _ = cnmf.pre_processing.get_noise_fft( Yr[:, :2000].astype(np.float32)) cnm.update_spatial(Yr) # update shapes a last time cnm.update_temporal(Yr) # update temporal trace a last time cnm.deconvolve() if use_parallel: cm.stop_server(dview=dview) print("Done @t = %s, (time passed: %s)" % (time.ctime(), str(time.time() - t_start))) print('Number of components left after merging:' + str(cnm.estimates.A.shape[-1])) ### ------------------- store results ------------------- ### ###%% store results in matlab array for further processing results = dict(A=cnm.estimates.A, C=cnm.estimates.C, S=cnm.estimates.S, Cn=Cn, b=cnm.estimates.b, f=cnm.estimates.f) savemat(svname, results) #hdf5storage.write(results, '.', svname, matlab_compatible=True) #cnm.estimates.coordinates = None #cnm.estimates.plot_contours(img=Cn, crd=None) #cnm.estimates.view_components(img=Cn) #plt.draw() #plt.pause(1) ### %% save only items that are needed to save disk-space #cnm.estimates = clear_cnm(cnm.estimates,retain=['A','C','S','b','f','YrA']) #cnm.save(svname_h5) print("Total time taken: " + str(time.time() - t_start)) os.remove(fname_memmap)
def run(batch_dir: str, UUID: str): logging.basicConfig( stream=sys.stdout, level=logging.DEBUG, format= "%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s" ) start_time = time() output = {'status': 0, 'output_info': ''} n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) file_path = os.path.join(batch_dir, UUID) filename = [file_path + '_input.tiff'] input_params = pickle.load(open(file_path + '.params', 'rb')) # If Ain is specified if input_params['do_cnmfe']: Ain = None item_uuid = input_params['cnmfe_kwargs'].pop('Ain') if item_uuid: print('>> Ain specified, looking for cnm-A file <<') parent_batch_dir = os.environ['CURR_BATCH_DIR'] item_out_file = os.path.join(parent_batch_dir, f'{item_uuid}.out') t0 = time() timeout = 60 while not os.path.isfile(item_out_file): print('>>> cnm-A not found, waiting for 15 seconds <<<') sleep(15) if time() - t0 > timeout: output.update({ 'status': 0, 'output_info': 'Timeout exceeding in waiting for Ain input file' }) raise TimeoutError( 'Timeout exceeding in waiting for Ain input file') if os.path.isfile(item_out_file): if json.load(open(item_out_file, 'r'))['status']: Ain_path = os.path.join(parent_batch_dir, item_uuid + '_results.hdf5') Ain = load_dict_from_hdf5(Ain_path)['estimates']['A'] print('>>> Found Ain file <<<') else: raise FileNotFoundError( '>>> Could not find specified Ain file <<<') print('*********** Creating Process Pool ***********') c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False, ignore_preexisting=True) try: print('Creating memmap') memmap_path = cm.save_memmap( filename, base_name=f'memmap-{UUID}', order='C', dview=dview, border_to_0=input_params['border_pix'], ) Yr, dims, T = cm.load_memmap(memmap_path) Y = Yr.T.reshape((T, ) + dims, order='F') if input_params['do_cnmfe']: gSig = input_params['cnmfe_kwargs']['gSig'][0] else: gSig = input_params['corr_pnr_kwargs']['gSig'] cn_filter, pnr = cm.summary_images.correlation_pnr(Y, swap_dim=False, gSig=gSig) if not input_params['do_cnmfe'] and input_params['do_corr_pnr']: pickle.dump(cn_filter, open(UUID + '_cn_filter.pikl', 'wb'), protocol=4) pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4) output_file_list = \ [ UUID + '_pnr.pikl', UUID + '_cn_filter.pikl', ] output.update({ 'output': UUID, 'status': 1, 'output_info': 'inspect correlation & pnr', 'output_files': output_file_list }) dview.terminate() for mf in glob(batch_dir + '/memmap-*'): try: os.remove(mf) except: # Windows doesn't like removing the memmaps pass end_time = time() processing_time = (end_time - start_time) / 60 output.update({'processing_time': processing_time}) json.dump(output, open(file_path + '.out', 'w')) return cnmf_params_dict = \ { "method_init": 'corr_pnr', "n_processes": n_processes, "only_init_patch": True, # for 1p "center_psf": True, # for 1p "normalize_init": False, # for 1p } cnmf_params_dict.update(**input_params['cnmfe_kwargs']) cnm = cnmf.CNMF( n_processes=n_processes, dview=dview, Ain=Ain, params=CNMFParams(params_dict=cnmf_params_dict), ) cnm.fit(Y) # DISCARD LOW QUALITY COMPONENTS cnm.params.set('quality', { 'use_cnn': False, **input_params['eval_kwargs'] }) cnm.estimates.evaluate_components(Y, cnm.params, dview=dview) out_filename = f'{UUID}_results.hdf5' cnm.save(out_filename) pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4) pickle.dump(cn_filter, open(UUID + '_cn_filter.pikl', 'wb'), protocol=4) output.update({ 'output': filename[:-5], 'status': 1, 'output_files': [ out_filename, UUID + '_pnr.pikl', UUID + '_cn_filter.pikl', ] }) except Exception as e: output.update({ 'status': 0, 'Y.shape': Y.shape, 'output_info': traceback.format_exc() }) dview.terminate() for mf in glob(batch_dir + '/memmap-*'): try: os.remove(mf) except: # Windows doesn't like removing the memmaps pass end_time = time() processing_time = (end_time - start_time) / 60 output.update({'processing_time': processing_time}) json.dump(output, open(file_path + '.out', 'w'))
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set to true to show movies and images fnames = ['data_endoscope.tif'] # filename to be processed fr = 10 # movie frame rate decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters do_motion_correction_nonrigid = False do_motion_correction_rigid = True # choose motion correction type gSig_filt = (3, 3) # size of filter, in general gSig (see below), # change this one if algorithm does not work max_shifts = (5, 5) # maximum allowed rigid shift strides = ( 48, 48 ) # start a new patch for pw-rigid motion correction every x pixels overlaps = (24, 24 ) # overlap between pathes (size of patch strides+overlaps) # for parallelization split the movies in num_splits chuncks across time # (make sure that length_movie/num_splits_to_process_rig>100) splits_rig = 10 splits_els = 10 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 #%% start the cluster try: cm.stop_server() # stop it if it was running except (): pass c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes= 24, # number of process to use, if you go out of memory try to reduce this one single_thread=False) #%% download demo file fnames = [download_demo(fnames[0])] filename_reorder = fnames #%% MOTION CORRECTION if do_motion_correction_nonrigid or do_motion_correction_rigid: # do motion correction rigid mc = motion_correct_oneP_rigid( fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, dview=dview, splits_rig=splits_rig, save_movie=not (do_motion_correction_nonrigid), border_nan='copy') new_templ = mc.total_template_rig plt.subplot(1, 2, 1) plt.imshow(new_templ) # % plot template plt.subplot(1, 2, 2) plt.plot(mc.shifts_rig) # % plot rigid shifts plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames') plt.ylabel('pixels') # borders to eliminate from movie because of motion correction bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) filename_reorder = mc.fname_tot_rig # do motion correction nonrigid if do_motion_correction_nonrigid: mc = motion_correct_oneP_nonrigid( fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, dview=dview, splits_rig=None, save_movie=True, # whether to save movie in memory mapped format new_templ=new_templ, # template to initialize motion correction border_nan='copy') filename_reorder = mc.fname_tot_els bord_px = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # create memory mappable file in the right order on the hard drive (C order) fname_new = cm.save_memmap(filename_reorder, base_name='memmap_', order='C', border_to_0=bord_px, dview=dview) # load memory mappable file Yr, dims, T = cm.load_memmap(fname_new) Y = Yr.T.reshape((T, ) + dims, order='F') #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None gSig = ( 3, 3 ) # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1 Ain = None # possibility to seed with predetermined binary masks merge_thresh = .7 # merging threshold, max correlation allowed rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80 stride_cnmf = 20 # amount of overlap between the patches in pixels # (keep it at least large as gSiz, i.e 4 times the neuron size gSig) tsub = 2 # downsampling factor in time for initialization, # increase if you have memory problems ssub = 1 # downsampling factor in space for initialization, # increase if you have memory problems # you can pass them here as boolean vectors low_rank_background = None # None leaves background of each patch intact, # True performs global low-rank approximation if gnb>0 gnb = 0 # number of background components (rank) if positive, # else exact ring model with following settings # gnb= 0: Return background as b and W # gnb=-1: Return full rank background B # gnb<-1: Don't return background nb_patch = 0 # number of background components (rank) per patch if gnb>0, # else it is set automatically min_corr = .8 # min peak value from correlation image min_pnr = 10 # min peak to noise ration from PNR image ssub_B = 2 # additional downsampling factor in space for background ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor # parameters for component evaluation min_SNR = 3 # adaptive way to set threshold on the transient size r_values_min = 0.85 # threshold on space consistency (if you lower more components # will be accepted, potentially with worst quality) opts = params.CNMFParams( dims=dims, fr=fr, decay_time=decay_time, method_init='corr_pnr', # use this for 1 photon k=K, gSig=gSig, gSiz=gSiz, merge_thresh=merge_thresh, p=p, tsub=tsub, ssub=ssub, rf=rf, stride=stride_cnmf, only_init_patch=True, # set it to True to run CNMF-E gnb=gnb, nb_patch=nb_patch, method_deconvolution='oasis', # could use 'cvxpy' alternatively low_rank_background=low_rank_background, update_background_components= True, # sometimes setting to False improve the results min_corr=min_corr, min_pnr=min_pnr, normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon ssub_B=ssub_B, ring_size_factor=ring_size_factor, del_duplicates=True, # whether to remove duplicates from initialization border_pix=bord_px) # number of pixels to not consider in the borders) #%% compute some summary images (correlation and peak to noise) # change swap dim if output looks weird, it is a problem with tiffile cn_filter, pnr = cm.summary_images.correlation_pnr(Y, gSig=gSig[0], swap_dim=False) # inspect the summary images and set the parameters inspect_correlation_pnr(cn_filter, pnr) # print parameters set above, modify them if necessary based on summary images print(min_corr) # min correlation of peak (from correlation image) print(min_pnr) # min peak to noise ratio #%% RUN CNMF ON PATCHES cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts) cnm.fit(Y) #%% DISCARD LOW QUALITY COMPONENTS cnm.params.set('quality', { 'min_SNR': min_SNR, 'rval_thr': r_values_min, 'use_cnn': False }) cnm.estimates.evaluate_components(Y, cnm.params, dview=dview) print(' ***** ') print('Number of total components: ', len(cnm.estimates.C)) print('Number of accepted components: ', len(cnm.estimates.idx_components)) #%% PLOT COMPONENTS cnm.dims = dims if display_images: cnm.estimates.plot_contours(img=cn_filter, idx=cnm.estimates.idx_components) cnm.estimates.view_components(Y, idx=cnm.estimates.idx_components) #%% MOVIES if display_images: # fully reconstructed movie cnm.estimates.play_movie(Y, q_max=99.9, magnification=2, include_bck=True, gain_res=10, bpx=bord_px) # movie without background cnm.estimates.play_movie(Y, q_max=99.9, magnification=2, include_bck=False, gain_res=4, bpx=bord_px) #%% STOP SERVER cm.stop_server(dview=dview)
def caimanAlign(fnames, frameRate=20): """ fnames: list of file path """ startSeconds = time.time() opts = caimanOptions.createParams(fnames, frameRate=frameRate) #start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened) if 'dview' in locals(): cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # # create a motion correction object with the parameters specified. #Note that the file is not loaded in memory mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # set mmap_file ''' tmpPath, tmpExt = os.path.splitext(fnames[0]) tmpPath += '.mmap' print('setting mc.mmap_file to tmpPath:', tmpPath) mc.mmap_file = tmpPath ''' #print('mc.mmap_file:', mc.mmap_file) # # perform motion correction # We will perform piecewise rigid motion correction using the NoRMCorre algorithm. # This has already been selected by setting pw_rigid=True when defining the parameters object. print(' performing motion correction ... please wait ...', fnames[0]) mc.motion_correct(save_movie=True) m_els = cm.load(mc.fname_tot_els) border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 print(' border_to_0:', border_to_0) # maximum shift to be used for trimming against NaNs # # memory mapping print(' memory mapping') print(' save_memmap mc.mmap_file:', mc.mmap_file) fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0, dview=dview) # exclude borders print(' mc.mmap_file:', mc.mmap_file) print(' fname_new:', fname_new) # make a copy of mmap file with a sane name # does not work, seems you can not 'copy' a mmap file like a normal file ''' tmpSavePath, tmpExt = os.path.splitext(fnames[0]) tmpSavePath += '.mmap' # need to be _aligned_ch1.tif shutil.copyfile(fname_new, tmpSavePath) ''' # save a _mmap.txt file with the name of the memory map file ''' tmpFile, tmpFilename = os.path.splitext(fnames[0]) tmpFile += '_mmap.txt' with open(tmpFile, 'w') as myTmpFile: myTmpFile.write(fname_new) ''' # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') #load frames in python format (T x X x Y) # save as .tif images_8bit = ((images - images.min()) / (images.ptp() / 255.0)).astype( np.uint8) # map the data range to 0 - 255 tmpSavePath, tmpExt = os.path.splitext(fnames[0]) tmpSavePath += '_aligned.tif' # need to be _aligned_ch1.tif print(' saving aligned .tif', images_8bit.shape, images_8bit.dtype, 'tmpSavePath:', tmpSavePath) tifffile.imsave(tmpSavePath, images_8bit, bigtiff=True) # shut down the cluster if 'dview' in locals(): cm.stop_server(dview=dview) # remove mmap file fname_new if os.path.isfile(fname_new): print(' removing mmap file:', fname_new) os.remove(fname_new) oneFile = mc.mmap_file[0] if os.path.isfile(oneFile): print(' removing mmap file mc.mmap_file[0]:', oneFile) os.remove(oneFile) # done stopSeconds = time.time() elapsedSeconds = round(stopSeconds - startSeconds, 3) elapsedMinutes = round(elapsedSeconds / 60, 3) print(f' caimanAlign() done in {elapsedMinutes} minutes')
def volspike(pars): """ Function for finding spikes of single neuron with given ROI in voltage imaging. Use function denoise_spikes to find spikes from one dimensional signal, and use ridge regression to find the best weight. Do these two steps iteratively to find best spike times. Args: pars: list fnames: str name of the memory mapping file in C order fr: int frame rate of the movie cell_n: int index of the cell processing ROIs: 3-d array all regions of interests weights: 3-d array spatial weights of different cells generated by previous data blocks as initialization args: dictionary context_size: int number of pixels surrounding the ROI to use as context censor_size: int number of pixels surrounding the ROI to censor from the background PCA; roughly the spatial scale of scattered/dendritic neural signals, in pixels visualize_ROI: boolean whether to visualize the region of interest inside the context region flip_signal: boolean whether to flip signal upside down for spike detection True for voltron, False for others hp_freq_pb: float high-pass frequency for removing photobleaching nPC_bg: int number of principle components used for background subtraction ridge_bg: float regularization strength for ridge regression in background removal hp_freq: float high-pass cutoff frequency to filter the signal after computing the trace clip: int maximum number of spikes for producing templates threshold_method: str adaptive_threshold or simple method for thresholding signals adaptive_threshold method threshold based on estimated peak distribution simple method threshold based on estimated noise level min_spikes: int minimal number of spikes to be detected pnorm: float a variable decides spike count chosen for adaptive threshold method threshold: float threshold for spike detection in simple threshold method The real threshold is the value multiplied by the estimated noise level sigmas: 1-d array spatial smoothing radius imposed on high-pass filtered movie only for finding weights n_iter: int number of iterations alternating between estimating spike times and spatial filters weight_update: str ridge or NMF for weight update do_plot: boolean if Ture, plot trace of signals and spiketimes, peak triggered average, histogram of heights in the last iteration do_cross_val: boolean whether to use cross validation to optimize regression regularization parameters sub_freq: float frequency for subthreshold extraction Returns: output: dictionary cell_n: int index of cell t: 1-d array trace without applying whitened matched filter ts: 1-d array trace after applying whitened matched filter t_rec: 1-d array reconstructed signal of the neuron t_sub: 1-d array subthreshold signal of the neuron spikes: 1-d array spike time of the neuron num_spikes: list number of spikes detected in each iteration low_spikes: boolean True if detected number spikes is less than min_spikes template: 1-d array temporal template of the neuron snr: float signal to noise ratio of the processed signal thresh: float threshold of the signal weights: 2-d array ridge regression coefficients for fitting reconstructed signal locality: boolean False if the maximum of spatial filter is not in the initial ROI context_coord: 2-d array boundary of context region in x,y coordinates mean_im: 1-d array mean of the signal in ROI after removing photobleaching, used for producing F0 F0: 1-d array baseline signal dFF: 1-d array scaled signal rawROI: dictionary including the result after the first spike extraction """ # load parameters fnames = pars[0] fr = pars[1] cell_n = pars[2] bw = pars[3] weights_init = pars[4] args = pars[5] window_length = fr * 0.02 # window length for temporal templates output = {} output['rawROI'] = {} print(f'Now processing cell number {cell_n}') # load the movie in C-order mermory mapping file Yr, dims, T = cm.load_memmap(fnames) if bw.shape == dims: images = np.reshape(Yr.T, [T] + list(dims), order='F') else: raise Exception('Dimensions of movie and ROIs do not accord') # extract the context region from the entire movie bwexp = dilation(bw, np.ones([args['context_size'], args['context_size']]), shift_x=True, shift_y=True) Xinds = np.where(np.any(bwexp > 0, axis=1) > 0)[0] Yinds = np.where(np.any(bwexp > 0, axis=0) > 0)[0] bw = bw[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] notbw = 1 - dilation(bw, disk(args['censor_size'])) data = np.array(images[:, Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1]) bw = (bw > 0) notbw = (notbw > 0) ref = np.median(data[:500, :, :], axis=0) bwexp[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] = True # visualize the ROI if args['visualize_ROI']: fig = plt.figure() plt.subplot(131) plt.imshow(ref) plt.axis('image') plt.xlabel('mean Intensity') plt.subplot(132) plt.imshow(bw) plt.axis('image') plt.xlabel('initial ROI') plt.subplot(133) plt.imshow(notbw) plt.axis('image') plt.xlabel('background') fig.suptitle('ROI selection') plt.show() # flip the signal if necessary if args['flip_signal'] == True: data = -data # remove the photobleaching effect by high-pass filtering the signal output['mean_im'] = np.mean(data, axis=0) data = np.reshape(data, (data.shape[0], -1)) data = data - np.mean(data, 0) data = data - np.mean(data, 0) #do again because of numeric issues data_hp = signal_filter(data.T, args['hp_freq_pb'], fr).T data_lp = data - data_hp # compute the initial trace if weights_init is None: t0 = np.nanmean(data_hp[:, bw.ravel()], 1) else: print('Reuse weights') t0 = np.matmul(data_hp, weights_init.ravel()) # reuse weights t0 = t0 - np.mean(t0) # remove any variance in trace that can be predicted from the background principal components data_svd = data_hp[:, notbw.ravel()] if data_svd.shape[1] < args['nPC_bg'] + 1: raise Exception( f'Too few pixels ({data_svd.shape[1]}) for background extraction (at least {args["nPC_bg"]} needed);' f'please decrease context_size and censor_size') Ub, Sb, Vb = svds(data_svd, args['nPC_bg']) alpha = args['nPC_bg'] * args[ 'ridge_bg'] # square of F-norm of Ub is equal to number of principal components reg = Ridge(alpha=alpha, fit_intercept=False, solver='lsqr').fit(Ub, t0) t0 = np.double(t0 - np.matmul(Ub, reg.coef_)) # spike detection for the initial trace ts, spikes, t_rec, templates, low_spikes, thresh = denoise_spikes( t0, window_length, fr, hp_freq=args['hp_freq'], clip=args['clip'], threshold_method=args['threshold_method'], pnorm=args['pnorm'], threshold=args['threshold'], min_spikes=args['min_spikes'], do_plot=False) output['rawROI']['t'] = t0.copy() output['rawROI']['ts'] = ts.copy() output['rawROI']['spikes'] = spikes.copy() if weights_init is None: output['rawROI']['weights'] = bw.copy() else: output['rawROI']['weights'] = weights_init.copy() output['rawROI']['t'] = output['rawROI']['t'] * np.mean( t0[output['rawROI']['spikes']]) / np.mean(output['rawROI']['t'][ output['rawROI']['spikes']]) # correct shrinkage output['rawROI']['templates'] = templates num_spikes = [spikes.shape[0]] # prebuild the regression matrix generate a predictor for ridge regression pred = np.empty_like(data_hp) pred[:] = data_hp pred = np.hstack( (np.ones((data_hp.shape[0], 1), dtype=np.single), np.reshape( movie.gaussian_blur_2D(np.reshape( pred, (data_hp.shape[0], ref.shape[0], ref.shape[1])), kernel_size_x=7, kernel_size_y=7, kernel_std_x=1.5, kernel_std_y=1.5, borderType=cv2.BORDER_REPLICATE), data_hp.shape))) # cross-validation of regularized regression parameters lambdamax = np.single(np.linalg.norm(pred[:, 1:], ord='fro')**2) lambdas = lambdamax * np.logspace(-4, -2, 3) if args['do_cross_val']: # need to add logging.warning('doing cross validation') raise Exception('cross validation option is not availble yet') else: s_max = 1 l_max = 2 sigma = args['sigmas'][s_max] recon = np.empty_like(data_hp) recon[:] = data_hp recon = np.hstack( (np.ones((data_hp.shape[0], 1), dtype=np.single), np.reshape( movie.gaussian_blur_2D( np.reshape(recon, (data_hp.shape[0], ref.shape[0], ref.shape[1])), kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1), kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1), kernel_std_x=sigma, kernel_std_y=sigma, borderType=cv2.BORDER_REPLICATE), data_hp.shape))) # refine weights and estimate spike times for several iterations for iteration in range(args['n_iter']): if iteration == args['n_iter'] - 1: do_plot = args['do_plot'] else: do_plot = False # update weights tr = np.single(t_rec.copy()) if args['weight_update'] == 'NMF': C = np.array([tr, np.ones_like(tr) ]) # constant baselines as 2nd component CCt = C.dot(C.T) CY = C.dot(recon[:, 1:]) A = np.maximum(np.linalg.inv(CCt).dot(CY), 0) for _ in range(5): for m in range(2): A[m] += (CY[m] - CCt[m].dot(A)) / CCt[m, m] if m == 0: A[m] = np.maximum(A[m], 0) weights = np.concatenate([[0], A[0]]) elif args['weight_update'] == 'ridge': Ri = Ridge(alpha=lambdas[l_max], fit_intercept=True, solver='lsqr') Ri.fit(recon, tr) weights = Ri.coef_ weights[0] = Ri.intercept_ # update the signal t = np.matmul(recon, weights) t = t - np.mean(t) # ridge regression to remove background components b = Ridge(alpha=alpha, fit_intercept=False, solver='lsqr').fit(Ub, t).coef_ t = t - np.matmul(Ub, b) # correct shrinkage weights = weights * np.mean(t0[spikes]) / np.mean(t[spikes]) t = np.double(t * np.mean(t0[spikes]) / np.mean(t[spikes])) # estimate spike times ts, spikes, t_rec, templates, low_spikes, thresh = denoise_spikes( t, window_length, fr, hp_freq=args['hp_freq'], clip=args['clip'], threshold_method=args['threshold_method'], pnorm=args['pnorm'], threshold=args['threshold'], min_spikes=args['min_spikes'], do_plot=do_plot) num_spikes.append(spikes.shape[0]) # compute SNR if len(spikes) > 0: t = t - np.median(t) selectSpikes = np.zeros(t.shape) selectSpikes[spikes] = 1 sgn = np.mean(t[selectSpikes > 0]) ff1 = -t * (t < 0) Ns = np.sum(ff1 > 0) noise = np.sqrt(np.divide(np.sum(ff1**2), Ns)) snr = sgn / noise else: snr = 0 # locality test matrix = np.matmul(np.transpose(pred[:, 1:]), t_rec) sigmax = np.sqrt(np.sum(np.multiply(pred[:, 1:], pred[:, 1:]), axis=0)) sigmay = np.sqrt(np.dot(t_rec, t_rec)) IMcorr = matrix / sigmax / sigmay maxCorrInROI = np.max(IMcorr[bw.ravel()]) if np.any(IMcorr[notbw.ravel()] > maxCorrInROI): locality = False else: locality = True # weights in the FOV weights = np.reshape(weights[1:], ref.shape, order='C') weights_FOV = np.zeros(images.shape[1:]) weights_FOV[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] = weights # subthreshold activity extraction t_sub = t.copy() - t_rec t_sub = signal_filter(t_sub, args['sub_freq'], fr, order=5, mode='low') # output output['cell_n'] = cell_n output['t'] = t output['ts'] = ts output['t_rec'] = t_rec output['t_sub'] = t_sub output['spikes'] = spikes output['low_spikes'] = low_spikes output['num_spikes'] = num_spikes output['templates'] = templates output['snr'] = snr output['thresh'] = thresh output['weights'] = weights_FOV output['locality'] = locality output['context_coord'] = np.transpose( np.vstack((Xinds[[0, -1]], Yinds[[0, -1]]))) output['F0'] = np.abs( np.nanmean( data_lp[:, bw.flatten()] + output['mean_im'][bw][np.newaxis, :], 1)) output['dFF'] = t / output['F0'] output['rawROI']['dFF'] = output['rawROI']['t'] / output['F0'] return output
def make_movie(self): """ Load memmaps and make the movie. """ Yr, dims, T = cm.load_memmap(self.memmap) self.movie = np.reshape(Yr.T, [T] + list(dims), order='F')
def volspike(pars): """ Main function for finding spikes of one single neuron with given ROI in voltage imaging. Using function denoiseSpikes to find spikes of one dimensional signal, using ridge regression to find the best spatial filters. Do these two steps iteratively to find best spike time. Args: pars: list fnames: str name of the memory map file fr: int cellN: int number of cell processing ROIs: 3-d array all region of interests weights: 3-d array spatial weights of different cells generated by previous data blocks as initialization args: dictionary doCrossVal: boolean whether to use cross validation to optimize regression regularization parameters doGlobalSubtract: boolean whether to subtract the signal which can be predicted by the entire video contextSize: int number of pixels surrounding the ROI to use as context censorSize: int number of pixels surrounding the ROI to censor from the background PCA; roughly the spatial scale of scattered/dendritic neural signals, in pixels nPC_bg: int number of principle components used for background subtraction tau_lp: int time window for lowpass filter (seconds); signals slower than this will be ignored tau_pred: int time window in seconds for high pass filtering to make predictor for regression sigmas: 1-d array spatial smoothing radius imposed on spatial filter nIter: int number of iterations alternating between estimating temporal and spatial filters localAlign: boolean globalAlign: boolean highPassRegression: boolean whether to regress on a high-passed version of the data. Slightly improves detection of spikes, but makes subthreshold unreliable use_ridge: boolean whether use ridge regression for removing background signal. If not, the algorithm will use linear regression Ridge_bg_coef: float regularization strength for ridge regression in background removal. Returns: output: dictionary rawROI: dictionary including the result after the first spike extraction bwexp: 2-d array expansion area of ROI, used for reconstructing signal meanIM: 1-d array trace after high-pass filter num_spikes: list number of spikes in each iteration passedLocalityTest: boolean False if the maximum of spatial filter is not in the initial ROI snr: float signal to noise ratio of the processed signal y: 1-d array processed signal of the video yFilt: 1-d array spike train of the signal ROIbw: 2-d array region of interest recons_signal: 1-d array reconstructed signal of the neuron spatialFilter: 2-d array spatial filter of the neuron falsePosRate: float possibility of misclassify noise as real spikes detectionRate: float possibility of real spikes being detected template: 1-d array spike template of the neuron spikeTimes: 1-d array spike time of the neuron thresh: float threshold of the signal F0: 1-d array initial signal dFF: 1-d array scaled signal bg_pc: 2-d array background principal components extracted from svd low_spk: boolean whether there are few spikes(less than 30) in the signal weights: 2-d array ridge regression coefficient for updating spatial filter cellN: int index of cell """ fnames = pars[0] sampleRate = pars[1] cellN = pars[2] bw = pars[3] weights_init = pars[4] args = pars[5] print('Now processing cell number {0}'.format(cellN)) doCrossVal = args['doCrossVal'] doGlobalSubtract = args['doGlobalSubtract'] contextSize = args['contextSize'] censorSize = args['censorSize'] nPC_bg = args['nPC_bg'] tau_lp = args['tau_lp'] tau_pred = args['tau_pred'] sigmas = args['sigmas'] nIter = args['nIter'] localAlign = args['localAlign'] globalAlign = args['globalAlign'] highPassRegression = args['highPassRegression'] use_Ridge = args['use_Ridge'] Ridge_bg_coef = args['Ridge_bg_coef'] windowLength = sampleRate * 0.02 # window length for spike templates output = {} output['rawROI'] = {} Yr, dims, T = cm.load_memmap(fnames) if bw.shape == dims: images = np.reshape(Yr.T, [T] + list(dims), order='F') elif bw.shape == dims[::-1]: images = np.reshape(Yr.T, [T] + list(dims), order='F').transpose([0, 2, 1]) else: print('size of ROI and video does not accrod') # extract relevant region and align bwexp = dilation(bw, np.ones([contextSize, contextSize]), shift_x=True, shift_y=True) Xinds = np.where(np.any(bwexp > 0, axis=1) > 0)[0] Yinds = np.where(np.any(bwexp > 0, axis=0) > 0)[0] bw = bw[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] notbw = 1 - dilation(bw, disk(censorSize)) data = np.array(images[:, Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1]) bw = (bw > 0) notbw = (notbw > 0) ref = np.median(data[:500, :, :], axis=0) bwexp[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] = True output['bwexp'] = bwexp # visualize ROI # fig = plt.figure() # plt.subplot(131);plt.imshow(ref);plt.axis('image');plt.xlabel('mean Intensity') # plt.subplot(132);plt.imshow(bw);plt.axis('image');plt.xlabel('initial ROI') # plt.subplot(133);plt.imshow(notbw);plt.axis('image');plt.xlabel('background') # fig.suptitle('ROI selection') # plt.show() output['meanIM'] = np.mean(data, axis=0) data = np.reshape(data, (data.shape[0], -1)) data = data - np.mean(data, 0) data = data - np.mean(data, 0) # remove low frequency components data_hp = highpassVideo(data.T, 1 / tau_lp, sampleRate).T """ data_pred = np.empty_like(data_hp) if highPassRegression: data_pred[:] = highpassVideo(data, 1 / tau_pred, sampleRate) else: data_pred[:] = data_hp """ # initial trace if weights_init is None: t = np.nanmean(data_hp[:, bw.ravel()], 1) else: t = -np.matmul(data_hp, weights_init[1:]) # weights are negative t = t - np.mean(t) # remove any variance in trace that can be predicted from the background principal components Ub, Sb, Vb = svds(data_hp[:, notbw.ravel()], nPC_bg) if use_Ridge: alpha = nPC_bg * Ridge_bg_coef # = np.single(np.linalg.norm(Ub, ord='fro') ** 2) * Ridge_bg_coef reg = Ridge(alpha=alpha, fit_intercept=False, solver='lsqr').fit(Ub, t) else: reg = LinearRegression(fit_intercept=False).fit(Ub, t) t = np.double(t - np.matmul(Ub, reg.coef_)) # find out spikes of initial trace Xspikes, spikeTimes, guessData, output['rawROI']['falsePosRate'], output['rawROI']['detectionRate'], \ output['rawROI']['templates'], low_spk, _ = denoiseSpikes(-t, windowLength, sampleRate, False, 100) Xspikes = -Xspikes output['rawROI']['X'] = t.copy() output['rawROI']['Xspikes'] = Xspikes.copy() output['rawROI']['spikeTimes'] = spikeTimes.copy() output['rawROI']['spatialFilter'] = bw.copy() output['rawROI']['X'] = output['rawROI']['X'] * np.mean( t[output['rawROI']['spikeTimes']]) / np.mean(output['rawROI']['X'][ output['rawROI']['spikeTimes']]) # correct shrinkage output['num_spikes'] = [spikeTimes.shape[0]] templates = output['rawROI']['templates'] selectSpikes = np.zeros(Xspikes.shape) selectSpikes[spikeTimes] = 1 sgn = np.mean(Xspikes[selectSpikes > 0]) noise = np.std(Xspikes[selectSpikes == 0]) snr = sgn / noise # prebuild the regression matrix # generate a predictor for ridge regression pred = np.empty_like(data_hp) pred[:] = data_hp pred = np.hstack( (np.ones((data_hp.shape[0], 1), dtype=np.single), np.reshape( movie.gaussian_blur_2D(np.reshape( pred, (data_hp.shape[0], ref.shape[0], ref.shape[1])), kernel_size_x=7, kernel_size_y=7, kernel_std_x=1.5, kernel_std_y=1.5, borderType=cv2.BORDER_REPLICATE), data_hp.shape))) # Cross-validation of regularized regression parameters lambdamax = np.single(np.linalg.norm(pred[:, 1:], ord='fro')**2) lambdas = lambdamax * np.logspace(-4, -2, 3) I0 = np.eye(pred.shape[1], dtype=np.single) I0[0, 0] = 0 if doCrossVal: # need to add print('doing cross validation') else: s_max = 1 l_max = 2 lambd = lambdas[l_max] sigma = sigmas[s_max] lambda_ix = l_max selectPred = np.ones(data_hp.shape[0]) if highPassRegression: selectPred[:np.int16(sampleRate / 2 + 1)] = 0 selectPred[-1 - np.int16(sampleRate / 2):] = 0 sigma = sigmas[s_max] """ pred = np.empty_like(data_pred) pred[:] = data_pred pred = np.hstack((np.ones((data_pred.shape[0], 1), dtype=np.single), np.reshape (movie.gaussian_blur_2D(np.reshape(pred, (data_pred.shape[0], ref.shape[0], ref.shape[1])), kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1), kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1), kernel_std_x=sigma, kernel_std_y=sigma, borderType=cv2.BORDER_REPLICATE), data_pred.shape))) """ recon = np.empty_like(data_hp) recon[:] = data_hp recon = np.hstack( (np.ones((data_hp.shape[0], 1), dtype=np.single), np.reshape( movie.gaussian_blur_2D( np.reshape(recon, (data_hp.shape[0], ref.shape[0], ref.shape[1])), kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1), kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1), kernel_std_x=sigma, kernel_std_y=sigma, borderType=cv2.BORDER_REPLICATE), data_hp.shape))) # Identify spatial filters with regularized regression for j in range(3 if args['weight_update'] == 'NMF' else 1): if j == 0: initialGuess = guessData[:] else: guessData = initialGuess[:] for iteration in range(nIter): doPlot = False if iteration == nIter - 1: doPlot = False # print('Identifying spatial filters') # print(iteration) gD = np.single(guessData[selectPred > 0]) if args['weight_update'] == 'NMF': C = np.array([gD, np.ones_like(gD) ]) # constant baselines as 2nd component CCt = C.dot(C.T) CY = C.dot(recon[:, 1:]) A = np.minimum(np.linalg.inv(CCt).dot(CY), 0) for _ in range(5): for m in range(2): A[m] += (CY[m] - CCt[m].dot(A)) / CCt[m, m] if m == 0: A[m] = np.minimum(A[m], 0) weights = np.concatenate([[0], A[0]]) else: Ri = Ridge(alpha=lambdas[l_max], fit_intercept=True, solver='lsqr') Ri.fit(recon, gD) weights = Ri.coef_ weights[0] = Ri.intercept_ X = np.matmul(recon, weights) X = X - np.mean(X) # for i in np.where(np.array([np.corrcoef(X, Ub[:,i])[0,1] for i in range(nPC_bg)]) > .4)[0]: # Ub[:, i] -= Ub[:, i].dot(X) / X.dot(X) * X spatialFilter = np.empty_like(weights) spatialFilter[:] = weights spatialFilter = movie.gaussian_blur_2D( np.reshape(spatialFilter[1:], ref.shape, order='C')[np.newaxis, :, :], kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1), kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1), kernel_std_x=sigma, kernel_std_y=sigma, borderType=cv2.BORDER_REPLICATE)[0] if use_Ridge: #alpha = np.single(np.linalg.norm(Ub, ord='fro') ** 2) * 0.01 b = Ridge(alpha=alpha, fit_intercept=False, solver='lsqr').fit(Ub, X).coef_ else: b = LinearRegression(fit_intercept=False).fit(Ub, X).coef_ """ if doPlot: plt.figure() plt.plot(X) plt.plot(np.matmul(Ub, b)) plt.title('Denoised trace vs background') plt.show() """ X = X - np.matmul(Ub, b) # correct shrinkage X = np.double(X * np.mean(t[spikeTimes]) / np.mean(X[spikeTimes])) # generate the new trace and the new denoised trace Xspikes, spikeTimes, guessData, falsePosRate, detectionRate, templates, _, thresh = denoiseSpikes( -X, windowLength, sampleRate, doPlot=False) selectSpikes = np.zeros(Xspikes.shape) selectSpikes[spikeTimes] = 1 sgn = np.mean(Xspikes[selectSpikes > 0]) noise = np.std(Xspikes[selectSpikes == 0]) snr = sgn / noise output['num_spikes'].append(spikeTimes.shape[0]) # ensure that the maximum of the spatial filter is within the ROI matrix = np.matmul(np.transpose(pred[:, 1:]), -guessData) sigmax = np.sqrt(np.sum(np.multiply(pred[:, 1:], pred[:, 1:]), axis=0)) sigmay = np.sqrt(np.dot(guessData, guessData)) IMcorr = matrix / sigmax / sigmay maxCorrInROI = np.max(IMcorr[bw.ravel()]) if np.any(IMcorr[notbw.ravel()] > maxCorrInROI): output['passedLocalityTest'] = False recon -= np.outer(recon.dot(weights), weights) / weights.dot(weights) else: output['passedLocalityTest'] = True break # compute SNR selectSpikes = np.zeros(Xspikes.shape) selectSpikes[spikeTimes] = 1 sgn = np.mean(Xspikes[selectSpikes > 0]) noise = np.std(Xspikes[selectSpikes == 0]) snr = sgn / noise output['snr'] = snr del pred del recon data_lp = data - data_hp # output output['y'] = X output['yFilt'] = -Xspikes output['ROI'] = np.transpose(np.vstack((Xinds[[0, -1]], Yinds[[0, -1]]))) output['ROIbw'] = bw output['recons_signal'] = guessData output['spatialFilter'] = -spatialFilter output['falsePosRate'] = falsePosRate output['detectionRate'] = detectionRate output['templates'] = templates output['spikeTimes'] = spikeTimes output['thresh'] = thresh output['F0'] = np.nanmean( data_lp[:, bw.flatten()] + output['meanIM'][bw][np.newaxis, :], 1) output['dFF'] = X / output['F0'] output['rawROI']['dFF'] = output['rawROI']['X'] / output['F0'] output['bg_pc'] = Ub # background components output['low_spk'] = low_spk output['weights'] = weights output['cellN'] = cellN return output
single_thread=False) # In[ ]: ## path to motion corrected tif file folder = '/projects/p30771/miniscope/data/GRIN011/1_24_2019/H10_M19_S59/TIFs/full_movie_caiman_analysis/' #mc_file = 'memmap__d1_480_d2_752_d3_1_order_C_frames_202_.mmap' memory_map_file = sys.argv[1] # In[4]: # load memory mappable file print('loading file:') print(memory_map_file) Yr, dims, T = cm.load_memmap(memory_map_file) images = Yr.T.reshape((T, ) + dims, order='F') # opts = params.CNMFParams(params_dict={}) # bord_px = 0 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None gSig = ( 3, 3 ) # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1 Ain = None # possibility to seed with predetermined binary masks merge_thresh = .7 # merging threshold, max correlation allowed
# Jan 42 folder = '/mnt/ceph/users/agiovann/ImagingData/LABELLING/Ben_labelled/Jan_Convnet/Jan42_2016_04_06_Green' file_mmap = 'Yr_d1_512_d2_512_d3_1_order_C_frames_3699_.mmap' ben_zip = '/mnt/ceph/users/agiovann/ImagingData/LABELLING/Ben_labelled/Jan_Convnet/Jan42_2016_04_06_Green/All_9.zip' princeton_zip = '/mnt/xfs1/home/agiovann/labeling/Jan42_exp4_001/regions/princeton_regions.mat' thresh_noise = -20 #%% import pylab as pl pl.ion() with np.load(os.path.join(folder,'results_analysis.npz')) as ld: locals().update(ld) A=scipy.sparse.coo_matrix(A) Yr, dims, T = cm.load_memmap(os.path.join(folder,file_mmap)) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') gSig = [8, 8] # expected half size of neurons final_frate=1 with np.load(os.path.join(folder,'results_blobs.npz')) as ld: print((list(ld.keys()))) locals().update(ld) masks_cnmf=masks[idx_blob] #%% crd = plot_contours(A.tocsc()[:, idx_components], Cn, thr=0.9) #%% masks_ben=cm.base.rois.nf_read_roi_zip(ben_zip,dims)
selected_rows = db.select(states_df, 'source_extraction', mouse=mouse_number, session=session, trial=6, is_rest=is_rest, cropping_v=cropping_version, motion_correction_v=motion_correction_version, alignment_v=0, source_extraction_v=1) row = selected_rows.iloc[1] #input_mmap_file_path = eval(row.loc['alignment_output'])['main'] input_mmap_file_path = eval(row.loc['motion_correction_output'])['main'] Yr, dims, T = cm.load_memmap(input_mmap_file_path) # logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.') images = Yr.T.reshape((T, ) + dims, order='F') images = cm.movie(images) output = eval(row.loc['source_extraction_output']) cnm_file_path = output['main'] cnm = load_CNMF(db.get_file(cnm_file_path)) W, b0 = cm.source_extraction.cnmf.initialization.compute_W( Yr, cnm.estimates.A.toarray(), cnm.estimates.C, cnm.estimates.dims, 1.4 * 5, ssub=2) cnm.estimates.W = W
def run_caiman_pipeline(data_path, opts, refit=False, component_evaluation=False, fr=14.913, rf=15, decay_time=0.6): '''Run the caiman pipeline out of the box, without using Agonia seeds. Parameters ---------- data_path : string path to folder containing the data opts : caiman object with option parameters refit : bool, optional if True re-run CNMF seeded on the selected components from the fitting component_evaluation : bool, optional if True filter components using shape and signal criteria fr : float frame rate of video rf : int half-size of the patches in pixels (in seeded is None) decay_time = float decay time of calcium indicator Returns ------- cnm : out of the box results of caiman detection cnm2 : refited-filtered results if refit, component_evaluations flags are True''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) opts_dict = { 'only_init': True, 'rf': rf, #15 'fr': fr, #14.913, 'decay_time': decay_time } #0.6 opts.change_params(opts_dict) if 'dview' in locals(): cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) if refit: cnm2 = cnm.refit(images, dview=dview) if component_evaluation: cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) #cnm2.estimates.select_components(use_object=True) if 'dview' in locals(): cm.stop_server(dview=dview) return cnm, cnm2
info = dict() info['experiment_title'] = experiment_title info['experiment_date'] = experiment_date info['animal_name'] = animal_name info['spatial_downsampling'] = spatial_downsampling info['downsample_subpath'] = downsample_subpath info['local_rootdir'] = local_rootdir session_fpaths = miniscope_file.list_session_dirs(local_miniscope_path, experiment_date, animal_name) info['session_fpaths'] = session_fpaths timestamp_files = [] session_lengths = [] for s_fpath in session_fpaths: vids_fpath = miniscope_file.list_vidfiles(s_fpath, vid_prefix) total_frames = 0 memmap_files = miniscope_file.get_memmap_files(s_fpath, doPwRigid, vid_prefix) for memmap_file in memmap_files: Yr, dim, T = cm.load_memmap(memmap_file, 'r') total_frames += T session_lengths.append(total_frames) timestamp_files.append(miniscope_file.get_timestamps_fpath(s_fpath)) info['session_lengths'] = session_lengths info['timestamp_files'] = timestamp_files with open(caiman_result_dir + '/session_info.yaml', 'w') as f: yaml.dump(info, f)
def signal_to_noise(data_path, agonia_th, ground_truth=None, neurofinder=False): '''Calculate signal to noise ratio of each box Parameters ---------- data_path : string path to folder containing the data agonia_th : float threshold for detection confidence for each box''' data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names( data_path) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load Caiman results cnm = cnmf.load_CNMF(results_caiman_path) # calculate the centers of the CaImAn factors centers = np.empty((cnm.estimates.A.shape[1], 2)) for i, factor in enumerate(cnm.estimates.A.T): centers[i] = center_of_mass(factor.toarray().reshape( cnm.estimates.dims, order='F')) with open(boxes_path, 'rb') as f: boxes = pickle.load(f) f.close() # keep only cells above confidence threshold boxes = boxes[boxes[:, 4] > agonia_th].astype('int') k = 0 for cell, box in enumerate(boxes): idx_factor = [ i for i, center in enumerate(centers) if center[0] > box[1] and center[0] < box[3] and center[1] > box[0] and center[1] < box[2] ] if not idx_factor: boxes = np.delete(boxes, cell - k, axis=0) k += 1 if ground_truth is not None: stats = ag.compute_stats(np.array(ground_truth), boxes, iou_threshold=0.5) stnr = [] #np.zeros(boxes.shape[0]) if neurofinder: for cell, box in enumerate(boxes): if ground_truth is not None: if cell in stats['true_positives'][:, 1]: box_trace = images[:, box[1]:box[3], box[0]:box[2]] box_trace_arr = np.array(images[:, box[1]:box[3], box[0]:box[2]]) box_trace_arr_cut = box_trace_arr.copy() box_trace_arr_cut[box_trace < 30] = np.nan mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2)) trace_flat = box_trace[box_trace > 30] noise_box = np.std( trace_flat[trace_flat < np.percentile(trace_flat, 25)]) amp_trace = max(mean_box) - min(mean_box) stnr.append(amp_trace / noise_box) else: box_trace = images[:, box[1]:box[3], box[0]:box[2]] box_trace_arr = np.array(images[:, box[1]:box[3], box[0]:box[2]]) box_trace_arr_cut = box_trace_arr.copy() box_trace_arr_cut[box_trace < 30] = np.nan mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2)) trace_flat = box_trace[box_trace > 30] noise_box = np.std( trace_flat[trace_flat < np.percentile(trace_flat, 25)]) amp_trace = max(mean_box) - min(mean_box) stnr.append(amp_trace / noise_box) else: for cell, box in enumerate(boxes): if ground_truth is not None: if cell in stats['true_positives'][:, 1]: box_trace = images[:, box[1]:box[3], box[0]:box[2]] mean_box = box_trace.mean(axis=(1, 2)) amp_trace = max(mean_box) - min(mean_box) noise_box = np.std(box_trace[box_trace < np.percentile( box_trace[box_trace != 0], 25)]) stnr.append(amp_trace / noise_box) else: box_trace = images[:, box[1]:box[3], box[0]:box[2]] mean_box = box_trace.mean(axis=(1, 2)) amp_trace = max(mean_box) - min(mean_box) noise_box = np.std(box_trace[ box_trace < np.percentile(box_trace[box_trace != 0], 25)]) stnr.append(amp_trace / noise_box) return np.array(stnr)
def run_component_evaluation(input_file, session_wise=False, equalization=False): sql = "SELECT source_extraction_session_wise,min_SNR,alignment_main,equalization_main,motion_correction_main,rval_thr,use_cnn FROM Analysis WHERE source_extraction_main=?" val = [ input_file, ] mycursor.execute(sql, val) myresult = mycursor.fetchall() data = [] aux = [] for x in myresult: aux = x for y in aux: data.append(y) if session_wise: input_mmap_file_path = data[2] elif equalization: input_mmap_file_path = data[3] else: input_mmap_file_path = data[4] parameters = {'min_SNR': data[1], 'rval_thr': data[5], 'use_cnn': data[6]} data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/session_wise/' if \ data[0] else os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/trial_wise/' sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,equalization_v,source_extraction_v,component_evaluation_v,input,home_path,decoding_main FROM Analysis WHERE source_extraction_main=?" val = [ input_file, ] mycursor.execute(sql, val) result = mycursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[10] == 0: data[10] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}" output_file_path = f'main/{file_name}.hdf5' sql1 = "UPDATE Analysis SET component_evaluation_main=?,component_evaluation_v=? WHERE source_extraction_main=? " val1 = [output_file_path, data[10], input_file] mycursor.execute(sql1, val1) else: data[10] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}" output_file_path = f'main/{file_name}.hdf5' sql2 = "INSERT INTO Analysis (component_evaluation_main,component_evaluation_v) VALUES (?,?)" val2 = [output_file_path, data[10]] mycursor.execute(sql2, val2) database.commit() output_file_path_full = data_dir + output_file_path # Load CNMF object cnm = load_CNMF(input_mmap_file_path) # Load the original movie Yr, dims, T = cm.load_memmap(input_mmap_file_path) images = Yr.T.reshape((T, ) + dims, order='F') # Set the parmeters cnm.params.set('quality', parameters) # Stop the cluster if one exists n_processes = psutil.cpu_count() try: cm.cluster.stop_server() except: pass # Start a new cluster c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_processes, # number of process to use, if you go out of memory try to reduce this one single_thread=False) # Evaluate components cnm.estimates.evaluate_components(images, cnm.params, dview=dview) logging.debug('Number of total components: ', len(cnm.estimates.C)) logging.debug('Number of accepted components: ', len(cnm.estimates.idx_components)) # Stop the cluster dview.terminate() # Save CNMF object cnm.save(output_file_path_full) return output_file_path
def main(): pass # For compatibility between running under Spyder and the CLI # %% start a cluster c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% set up some parameters fnames = [ os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif') ] # file(s) to be analyzed is_patches = True # flag for processing in patches or not fr = 10 # approximate frame rate of data decay_time = 5.0 # length of transient if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 10 # half size of each patch stride = 4 # overlap between patches K = 4 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = 30 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order params_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'rf': rf, 'stride': stride, 'K': K, 'gSig': gSig, 'merge_thr': merge_thresh, 'p': p, 'nb': gnb } opts = params.CNMFParams(params_dict=params_dict) # %% Now RUN CaImAn Batch (CNMF) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit_file() # %% plot contour plots of components Cns = local_correlations_movie_offline(fnames[0], remove_baseline=True, swap_dim=False, window=1000, stride=1000, winSize_baseline=100, quantil_min_baseline=10, dview=dview) Cn = Cns.max(axis=0) cnm.estimates.plot_contours(img=Cn) # %% load memory mapped file Yr, dims, T = cm.load_memmap(cnm.mmap_file) images = np.reshape(Yr.T, [T] + list(dims), order='F') # %% refit cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) min_SNR = 2 # peak SNR for accepted components (if above this, acept) rval_thr = 0.85 # space correlation threshold (if above this, accept) use_cnn = True # use the CNN classifier min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': use_cnn, 'min_cnn_thr': min_cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% visualize selected and rejected components cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% visualize selected components cnm2.estimates.view_components(images, idx=cnm2.estimates.idx_components, img=Cn) #%% only select high quality components (destructive) # cnm2.estimates.select_components(use_object=True) # cnm2.estimates.plot_contours(img=Cn) #%% save results cnm2.estimates.Cn = Cn cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') # %% play movie with results (original, reconstructed, amplified residual) cnm2.estimates.play_movie(images, magnification=4) # %% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file)
def load_images(memmap_fpath): # load memory mappable file Yr, dims, T = cm.load_memmap(memmap_fpath) images = Yr.T.reshape((T,) + dims, order='F') return images
def run(work_dir: str, UUID: str, save_temp_files: str): logging.basicConfig( stream=sys.stdout, level=logging.DEBUG, format= "%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s" ) start_time = time() batch_dir = os.environ['CURR_BATCH_DIR'] save_temp_files = bool(int(save_temp_files)) output = {'status': 0, 'output_info': ''} n_processes = int(os.environ['_MESMERIZE_N_THREADS']) filepath = os.path.join(work_dir, UUID) imgpath = f'{filepath}_input.tiff' input_params = pickle.load(open(f'{filepath}.params', 'rb')) print('******** Creating process pool *********') c, dview, n_processes = setup_cluster(backend='local', n_processes=n_processes, single_thread=False, ignore_preexisting=True) try: if input_params['use_memmap']: memmap_uuid = input_params['memmap_uuid'] memmap_batchdir = glob( os.path.join(batch_dir, f'memmap-{memmap_uuid}*.mmap')) # Check batch dir if len(memmap_batchdir) > 0: memmap_path = memmap_batchdir[0] print( f'********** Found existing memmap in batch dir: {memmap_path} ********** ' ) # copy to work dir if not os.path.samefile(batch_dir, work_dir): print('**** Copying memmap to work dir ****') shutil.copy(memmap_path, work_dir) memmap_path = glob( os.path.join(work_dir, f'memmap-{memmap_uuid}*.mmap'))[0] else: # remake the memmap with the same UUID so that future batch items can rely on it print( '********** Memmap not found, re-making memmap with the same UUID **********' ) memmap_path = cm.save_memmap([imgpath], base_name=f'memmap-{memmap_uuid}', is_3D=True, order='C', dview=dview) else: print('********** Making memmap **********') memmap_path = cm.save_memmap([imgpath], base_name=f'memmap-{UUID}', is_3D=True, order='C', dview=dview) print(f'Using memmap:\n{memmap_path}') print('********** Loading memmap **********') Yr, dims, T = cm.load_memmap(memmap_path) Y = np.reshape(Yr, dims + (T, ), order='F') images = np.reshape(Yr.T, [T] + list(dims), order='F') if input_params['use_patches']: cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, only_init_patch=True, **input_params['cnmf_kwargs']) else: cnm = cnmf.CNMF(n_processes, dview=dview, **input_params['cnmf_kwargs']) cnm.fit(images) print('Number of components:' + str(cnm.estimates.A.shape[-1])) cnm.params.change_params( params_dict={ **input_params['eval_kwargs'], 'use_cnn': False }) cnm.estimates.evaluate_components(images, cnm.params, dview=dview) if input_params['refit']: cnm.params.set('temporal', {'p': input_params['cnmf_kwargs']['p']}) cnm_ = cnm.refit(images) else: cnm_ = cnm out_filename = f'{UUID}_results.hdf5' cnm_.save(out_filename) output_files = [out_filename] # Save the memmap if save_temp_files: print("***** Keeping memmap file *****") # copy to batch dir if batch_dir != work_dir if not os.path.samefile(batch_dir, work_dir): print("***** Copying memmap file to batch dir *****") shutil.copy(memmap_path, batch_dir) # Delete the memmap from the work dir if not os.path.samefile(batch_dir, work_dir): print("***** Deleting memmap files from work dir *****") try: os.remove(memmap_path) except: pass output.update({ 'output': UUID, 'status': 1, 'output_files': output_files, 'saved_memmap': save_temp_files }) except Exception as e: output.update({'status': 0, 'output_info': traceback.format_exc()}) cm.stop_server(dview=dview) end_time = time() processing_time = (end_time - start_time) / 60 output.update({'processing_time': processing_time}) json.dump(output, open(filepath + '.out', 'w'))
def select_corr_pnr_threshold(mouse_row, parameters_source_extraction): ''' Plots the summary images correlation and pnr. Also the pointwise product between them (used in Caiman paper Zhou et al 2018) :param mouse_row: :param parameters_source_extraction: parameters that will be used for source extraction. the relevant parameter here are min_corr and min_pnr because the source extraction algorithm is initialized (initial cell templates) in all values that surpasses that threshold :return: max_combined, max_pnr, max_corr: threshold for corr*pnr, and corresponding values of corr and pnr ''' input_mmap_file_path = eval( mouse_row.loc['motion_correction_output'])['main'] # Load memory mappable input file if os.path.isfile(input_mmap_file_path): Yr, dims, T = cm.load_memmap(input_mmap_file_path) # logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.') images = Yr.T.reshape((T, ) + dims, order='F') else: logging.warning( f'{mouse_row.name} .mmap file does not exist. Cancelling') # Determine output paths step_index = db.get_step_index('motion_correction') data_dir = 'data/interim/source_extraction/trial_wise/' # Check if the summary images are already there gSig = parameters_source_extraction['gSig'][0] corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path( mouse_row.name, gSig_abs=(gSig, gSig)) if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path): # Already computed summary images logging.info(f'{mouse_row.name} Already computed summary images') cn_filter = np.load(corr_npy_file_path) pnr = np.load(pnr_npy_file_path) else: # Compute summary images t0 = datetime.datetime.today() logging.info(f'{mouse_row.name} Computing summary images') cn_filter, pnr = cm.summary_images.correlation_pnr( images[::1], gSig=parameters_source_extraction['gSig'][0], swap_dim=False) # Saving summary images as npy files corr_npy_file_path = data_dir + f'meta/corr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy' pnr_npy_file_path = data_dir + f'meta/pnr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy' with open(corr_npy_file_path, 'wb') as f: np.save(f, cn_filter) with open(pnr_npy_file_path, 'wb') as f: np.save(f, pnr) combination = cn_filter * pnr # this is as defined in Zhou et al 2018 (definition of R, P and L, eq 14) max_combined = np.argmax(combination) row = int(np.floor(max_combined / cn_filter.shape[1])) column = int(max_combined - row * cn_filter.shape[1]) max_corr = cn_filter[row, column] max_pnr = pnr[row, column] return max_combined, max_corr, max_pnr
def initialize_movie_online(Y, K, gSig, rf, stride, base_name, p=1, merge_thresh=0.95, rval_thr_online=0.9, thresh_fitness_delta_online=-30, thresh_fitness_raw_online=-50, rval_thr_init=.5, thresh_fitness_delta_init=-20, thresh_fitness_raw_init=-20, rval_thr_refine=0.95, thresh_fitness_delta_refine=-100, thresh_fitness_raw_refine=-100, final_frate=10, Npeaks=10, single_thread=True, dview=None, n_processes=None): """ Initialize movie using CNMF on minibatch. See CNMF parameters """ _, d1, d2 = Y.shape dims = (d1, d2) Yr = Y.to_2D().T # merging threshold, max correlation allowed # order of the autoregressive system #T = Y.shape[0] base_name = base_name + '.mmap' fname_new = Y.save(base_name, order='C') #% Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') Cn2 = cm.local_correlations(Y) # pl.imshow(Cn2) #% #% RUN ALGORITHM ON PATCHES # pl.close('all') cnm_init = cm.source_extraction.cnmf.CNMF( n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, Ain=None, rf=rf, stride=stride, method_deconvolution='oasis', skip_refinement=False, normalize_init=False, options_local_NMF=None, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr_online, thresh_fitness_delta=thresh_fitness_delta_online, thresh_fitness_raw=thresh_fitness_raw_online, batch_update_suff_stat=True, max_comp_update_shape=5) cnm_init = cnm_init.fit(images) A_tot = cnm_init.A C_tot = cnm_init.C YrA_tot = cnm_init.YrA b_tot = cnm_init.b f_tot = cnm_init.f print(('Number of components:' + str(A_tot.shape[-1]))) #% traces = C_tot + YrA_tot # traces_a=traces-scipy.ndimage.percentile_filter(traces,8,size=[1,np.shape(traces)[-1]/5]) # traces_b=np.diff(traces,axis=1) fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = cm.components_evaluation.evaluate_components( Y, traces, A_tot, C_tot, b_tot, f_tot, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= rval_thr_init)[0] idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_init)[0] idx_components_delta = np.where( fitness_delta < thresh_fitness_delta_init)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components) print(('Keeping ' + str(len(idx_components)) + ' and discarding ' + str(len(idx_components_bad)))) A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] #% cnm_refine = cm.source_extraction.cnmf.CNMF( n_processes, method_init='greedy_roi', k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, rf=None, stride=None, p=p, dview=dview, Ain=A_tot, Cin=C_tot, f_in=f_tot, method_deconvolution='oasis', skip_refinement=True, normalize_init=False, options_local_NMF=None, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr_refine, thresh_fitness_delta=thresh_fitness_delta_refine, thresh_fitness_raw=thresh_fitness_raw_refine, batch_update_suff_stat=True, max_comp_update_shape=5) cnm_refine = cnm_refine.fit(images) #% A, C, b, f, YrA, sn = cnm_refine.A, cnm_refine.C, cnm_refine.b, cnm_refine.f, cnm_refine.YrA, cnm_refine.sn #% final_frate = 10 Npeaks = 10 traces = C + YrA fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \ cm.components_evaluation.evaluate_components(Y, traces, A, C, b, f, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= rval_thr_refine)[0] idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_refine)[0] idx_components_delta = np.where( fitness_delta < thresh_fitness_delta_refine)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components) print(' ***** ') print((len(traces))) print((len(idx_components))) #% cnm_refine.idx_components = idx_components cnm_refine.idx_components_bad = idx_components_bad cnm_refine.r_values = r_values cnm_refine.fitness_raw = fitness_raw cnm_refine.fitness_delta = fitness_delta cnm_refine.Cn2 = Cn2 #% # cnm_init.dview = None # save_object(cnm_init,fls[0][:-4]+ '_DS_' + str(ds)+ '_init.pkl') return cnm_refine, Cn2, fname_new
def run_source_extraction(row, parameters, dview, session_wise = False): ''' This is the function for source extraction. Its goal is to take in a .mmap file, perform source extraction on it using cnmf-e and save the cnmf object as a .pkl file. Args: row: pd.DataFrame object The row corresponding to the analysis state to be source extracted. Returns: row: pd.DataFrame object The row corresponding to the source extracted analysis state. ''' step_index = 5 row_local = row.copy() row_local.loc['source_extraction_parameters'] = str(parameters) row_local = db.set_version_analysis('source_extraction',row_local,session_wise) index = row_local.name # Determine input path if parameters['session_wise']: input_mmap_file_path = eval(row_local.loc['alignment_output'])['main'] if parameters['equalization']: input_mmap_file_path =eval(row_local['equalization_output'])['main'] else: input_mmap_file_path = eval(row_local.loc['motion_correction_output'])['main'] if not os.path.isfile(input_mmap_file_path): logging.error('Input file does not exist. Cancelling.') return row_local # Determine output paths file_name = db.create_file_name(step_index, index) if parameters['session_wise']: data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/session_wise/' else: data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/trial_wise/' output_file_path = data_dir + f'main/{file_name}.hdf5' # Create a dictionary with parameters output = { 'main': output_file_path, 'meta':{ 'analysis' : { 'analyst' : os.environ['ANALYST'], 'date' : datetime.datetime.today().strftime("%m-%d-%Y"), 'time' : datetime.datetime.today().strftime("%H:%M:%S"), }, 'duration': {} } } # Load memmory mappable input file if os.path.isfile(input_mmap_file_path): Yr, dims, T = cm.load_memmap(input_mmap_file_path) # logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.') images = Yr.T.reshape((T,) + dims, order='F') else: logging.warning(f'{index} .mmap file does not exist. Cancelling') return row_local # SOURCE EXTRACTION # Check if the summary images are already there corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(index, gSig_abs = parameters['gSig'][0]) if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path): # Already computed summary images logging.info(f'{index} Already computed summary images') cn_filter = np.load(corr_npy_file_path) pnr = np.load(pnr_npy_file_path) else: # Compute summary images t0 = datetime.datetime.today() logging.info(f'{index} Computing summary images') cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig = parameters['gSig'][0], swap_dim=False) dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes output['meta']['duration']['summary_images'] = dt logging.info(f'{index} Computed summary images. dt = {dt} min') # Saving summary images as npy files gSig = parameters['gSig'][0] corr_npy_file_path = data_dir + f'/meta/corr/{db.create_file_name(3, index)}_gSig_{gSig}.npy' pnr_npy_file_path = data_dir + f'/meta/pnr/{db.create_file_name(3, index)}_gSig_{gSig}.npy' with open(corr_npy_file_path, 'wb') as f: np.save(f, cn_filter) with open(pnr_npy_file_path, 'wb') as f: np.save(f, pnr) # Store the paths in the meta dictionary output['meta']['corr'] = {'main': corr_npy_file_path, 'meta': {}} output['meta']['pnr'] = {'main': pnr_npy_file_path, 'meta': {}} # Calculate min, mean, max value for cn_filter and pnr corr_min, corr_mean, corr_max = cn_filter.min(), cn_filter.mean(), cn_filter.max() output['meta']['corr']['meta'] = {'min': corr_min, 'mean': corr_mean, 'max': corr_max} pnr_min, pnr_mean, pnr_max = pnr.min(), pnr.mean(), pnr.max() output['meta']['pnr']['meta'] = {'min': pnr_min, 'mean': pnr_mean, 'max': pnr_max} # If min_corr and min_pnr are specified via a linear equation, calculate # this value if type(parameters['min_corr']) == list: min_corr = parameters['min_corr'][0]*corr_mean + parameters['min_corr'][1] parameters['min_corr'] = min_corr logging.info(f'{index} Automatically setting min_corr = {min_corr}') if type(parameters['min_pnr']) == list: min_pnr = parameters['min_pnr'][0]*pnr_mean + parameters['min_pnr'][1] parameters['min_pnr'] = min_pnr logging.info(f'{index} Automatically setting min_pnr = {min_pnr}') # Set the parameters for caiman opts = params.CNMFParams(params_dict = parameters) # SOURCE EXTRACTION logging.info(f'{index} Performing source extraction') t0 = datetime.datetime.today() n_processes = psutil.cpu_count() logging.info(f'{index} n_processes: {n_processes}') cnm = cnmf.CNMF(n_processes = n_processes, dview = dview, params = opts) cnm.fit(images) cnm.estimates.dims = dims # Store the number of neurons output['meta']['K'] = len(cnm.estimates.C) # Calculate the center of masses cnm.estimates.center = caiman.base.rois.com(cnm.estimates.A, images.shape[1], images.shape[2]) # Save the cnmf object as a hdf5 file logging.info(f'{index} Saving cnmf object') cnm.save(output_file_path) dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes output['meta']['duration']['source_extraction'] = dt logging.info(f'{index} Source extraction finished. dt = {dt} min') # Write necessary variables in row and return row_local.loc['source_extraction_parameters'] = str(parameters) row_local.loc['source_extraction_output'] = str(output) return row_local
def plot_corr_pnr(mouse_row, parameters_source_extraction): ''' Plots the summary images correlation and pnr. Also the pointwise product between them (used in Caiman paper Zhou et al 2018) :param mouse_row: :param parameters_source_extraction: parameters that will be used for source extraction. the relevant parameter here are min_corr and min_pnr because the source extraction algorithm is initialized (initial cell templates) in all values that surpasses that threshold :return: figure ''' input_mmap_file_path = eval(mouse_row.loc['motion_correction_output'])['main'] # Load memory mappable input file if os.path.isfile(input_mmap_file_path): Yr, dims, T = cm.load_memmap(input_mmap_file_path) # logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.') images = Yr.T.reshape((T,) + dims, order='F') else: logging.warning(f'{mouse_row.name} .mmap file does not exist. Cancelling') # Determine output paths step_index = db.get_step_index('motion_correction') data_dir = 'data/interim/source_extraction/trial_wise/' # Check if the summary images are already there gSig = parameters_source_extraction['gSig'][0] corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(mouse_row.name, gSig_abs=(gSig, gSig)) if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path): # Already computed summary images logging.info(f'{mouse_row.name} Already computed summary images') cn_filter = np.load(corr_npy_file_path) pnr = np.load(pnr_npy_file_path) else: # Compute summary images t0 = datetime.datetime.today() logging.info(f'{mouse_row.name} Computing summary images') cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig=parameters_source_extraction['gSig'][0], swap_dim=False) # Saving summary images as npy files corr_npy_file_path = data_dir + f'meta/corr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy' pnr_npy_file_path = data_dir + f'meta/pnr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy' with open(corr_npy_file_path, 'wb') as f: np.save(f, cn_filter) with open(pnr_npy_file_path, 'wb') as f: np.save(f, pnr) fig = plt.figure(figsize=(15, 15)) min_corr = round(parameters_source_extraction['min_corr'], 2) min_pnr = round(parameters_source_extraction['min_pnr'], 1) max_corr = round(cn_filter.max(), 2) max_pnr= 20 # continuous cmap = 'viridis' fig, axes = plt.subplots(1, 3, sharex=True) corr_fig = axes[0].imshow(np.clip(cn_filter,min_corr,max_corr), cmap=cmap) axes[0].set_title('Correlation') fig.colorbar(corr_fig, ax=axes[0]) pnr_fig = axes[1].imshow(np.clip(pnr,min_pnr,max_pnr), cmap=cmap) axes[1].set_title('PNR') fig.colorbar(pnr_fig, ax=axes[1]) combined = cn_filter*pnr max_combined = 10 min_combined = np.min(combined) corr_pnr_fig = axes[2].imshow(np.clip(cn_filter*pnr,min_combined,max_combined), cmap=cmap) axes[2].set_title('Corr * PNR') fig.colorbar(corr_pnr_fig, ax=axes[2]) fig_dir = 'data/interim/source_extraction/trial_wise/meta/' fig_name= fig_dir + f'figures/corr_pnr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.png' fig.savefig(fig_name) return fig
def run(batch_dir: str, UUID: str): output = {'status': 0, 'output_info': ''} n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) file_path = batch_dir + '/' + UUID filename = [file_path + '.tiff'] input_params = pickle.load(open(file_path + '.params', 'rb')) fr = input_params['fr'] p = input_params['p'] gnb = input_params['gnb'] merge_thresh = input_params['merge_thresh'] rf = input_params['rf'] stride_cnmf = input_params['stride_cnmf'] K = input_params['k'] gSig = input_params['gSig'] gSig = [gSig, gSig] min_SNR = input_params['min_SNR'] rval_thr = input_params['rval_thr'] cnn_thr = input_params['cnn_thr'] decay_time = input_params['decay_time'] bord_px = input_params['bord_px'] refit = input_params['refit'] print('*********** Creating Process Pool ***********') c, dview, np = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False) try: print('Creating memmap') fname_new = cm.save_memmap_each( filename, base_name='memmap_' + UUID, order='C', border_to_0=bord_px, dview=dview) fname_new = cm.save_memmap_join(fname_new, base_name='memmap_' + UUID, dview=dview) Yr, dims, T = cm.load_memmap(fname_new) Y = Yr.T.reshape((T,) + dims, order='F') cnm = cnmf.CNMF(n_processes=n_processes, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init='greedy_roi', alpha_snmf=None, only_init_patch=False, gnb=gnb, border_pix=bord_px) cnm.fit(Y) idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_lowest=cnn_thr) if refit: A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=n_processes, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(Y) cnmA = cnm2.A cnmb = cnm2.b cnmC = cnm2.C cnm_f = cnm2.f cnmYrA = cnm2.YrA else: cnmA = cnm.A cnmb = cnm.b cnmC = cnm.C cnm_f = cnm.f cnmYrA = cnm.YrA pickle.dump(Yr, open(UUID + '_Yr.pikl', 'wb'), protocol=4) pickle.dump(cnmA, open(UUID + '_cnm-A.pikl', 'wb'), protocol=4) pickle.dump(cnmb, open(UUID + '_cnm-b.pikl', 'wb'), protocol=4) pickle.dump(cnmC, open(UUID + '_cnm-C.pikl', 'wb'), protocol=4) pickle.dump(cnm_f, open(UUID + '_cnm-f.pikl', 'wb'), protocol=4) pickle.dump(idx_components, open(UUID + '_idx_components.pikl', 'wb'), protocol=4) pickle.dump(cnmYrA, open(UUID + '_cnm-YrA.pikl', 'wb'), protocol=4) pickle.dump(dims, open(UUID + '_dims.pikl', 'wb'), protocol=4) output_file_list = [UUID + '_cnm-A.pikl', UUID + '_Yr.pikl', UUID + '_cnm-b.pikl', UUID + '_cnm-C.pikl', UUID + '_cnm-f.pikl', UUID + '_idx_components.pikl', UUID + '_cnm-YrA.pikl', UUID + '_dims.pikl', UUID + '.out' ] output.update({'output': UUID, 'status': 1, 'output_files': output_file_list }) except Exception: output.update({'status': 0, 'output_info': traceback.format_exc()}) dview.terminate() for mf in glob(batch_dir + '/memmap_*'): os.remove(mf) json.dump(output, open(file_path + '.out', 'w'))