def inspect_results(images, cnm, is_3d=False): """Inspect results by plotting contours of identified components against correlation image. The results of the algorithm are stored in the object cnm.estimates.""" # Plot contours of found components if is_3d: cn = cm.local_correlations(images, swap_dim=False) cn[np.isnan(cn)] = 0 return cnm, cn cn = cm.local_correlations(images.transpose(1, 2, 0)) cn[np.isnan(cn)] = 0 cnm.estimates.plot_contours_nb(img=cn) return cnm, cn
def view_corr(self): self.corr_images = [] fig, axes = plt.subplots(1, self.planes, constrained_layout=True) for i,ax in enumerate(axes): image = self.motion_corrected_images[i] corr_im = cm.local_correlations(image, swap_dim=False) self.corr_images.append(corr_im) ax.imshow(image) ax.set_aspect('equal', 'box') ax.axis('off') ax.set_title(f'Plane {i}') fig.suptitle('Correlation Image')
def output_cnmf(self): # , batch_dir, UUID, viewer_ref): pass filename = self.batch_dir + '/' + str(self.UUID) self.get_cnmf_results() Yr = pickle.load(open(filename + '_Yr.pikl', 'rb')) cnmb = pickle.load(open(filename + '_cnm-b.pikl', 'rb')) cnm_f = pickle.load(open(filename + '_cnm-f.pikl', 'rb')) cnmYrA = pickle.load(open(filename + '_cnm-YrA.pikl', 'rb')) img = tifffile.imread(self.batch_dir + '/' + str(self.UUID) + '.tiff') Cn = cm.local_correlations(img.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 self.visualization = cm.utils.visualization.view_patches_bar(Yr, self.cnmA[:, self.idx_components], self.cnmC[self.idx_components], cnmb, cnm_f, self.dims[0], self.dims[1], YrA=cnmYrA[self.idx_components], img=Cn)
t2 = time.time() - t1 #%% LOAD MEMMAP FILE # fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap' Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') m_images = cm.movie(images) #%% checks on movies (might take time if large!) # if np.min(images) < 0: # raise Exception('Movie too negative, add_to_movie should be larger') # if np.sum(np.isnan(images)) > 0: # raise Exception('Movie contains nan! You did not remove enough borders') #%% correlation image Cn = cm.local_correlations(Y) Cn[np.isnan(Cn)] = 0 pl.imshow(Cn, cmap='gray', vmax=.55) #%% some parameter settings # order of the autoregressive fit to calcium imaging in general one (slow gcamps) or two (fast gcamps fast scanning) p = params_movie['p'] # merging threshold, max correlation allowed merge_thresh = params_movie['merge_thresh'] # half-size of the patches in pixels. rf=25, patches are 50x50 rf = params_movie['rf'] # amounpl.it of overlap between the patches in pixels stride_cnmf = params_movie['stride_cnmf'] # number of components per patch K = params_movie['K'] # if dendritic. In this case you need to set init_method to sparse_nmf is_dendrites = params_movie['is_dendrites']
def gen_data(dims=(48, 48), N=10, sig=(3, 3), tau=1., noise=.3, T=2000, framerate=30, firerate=.5, seed=3, cmap=False, truncate=np.exp(-2), difference_of_Gaussians=True, fluctuating_bkgrd=[50, 300]): bkgrd = 10 # fluorescence baseline np.random.seed(seed) boundary = 4 M = int(N * 1.5) # centers = boundary + (np.array(GeneralizedHalton(2, seed).get(M)) * # (np.array(dims) - 2 * boundary)).astype('uint16') centers = boundary + (np.random.rand(M, 2) * (np.array(dims) - 2 * boundary)).astype('uint16') trueA = np.zeros(dims + (M,), dtype='float32') for i in range(M): trueA[tuple(centers[i]) + (i,)] = 1. if difference_of_Gaussians: q = .75 for n in range(M): s = (.67 + .33 * np.random.rand(2)) * np.array(sig) tmp = gaussian_filter(trueA[:, :, n], s) trueA[:, :, n] = np.maximum(tmp - gaussian_filter(trueA[:, :, n], q * s) * q**2 * (.2 + .6 * np.random.rand()), 0) else: for n in range(M): s = [ss * (.75 + .25 * np.random.rand()) for ss in sig] trueA[:, :, n] = gaussian_filter(trueA[:, :, n], s) trueA = trueA.reshape((-1, M), order='F') trueA *= (trueA >= trueA.max(0) * truncate) trueA /= np.linalg.norm(trueA, 2, 0) keep = np.ones(M, dtype=bool) overlap = trueA.T.dot(trueA) - np.eye(M) while keep.sum() > N: keep[np.argmax(overlap * np.outer(keep, keep)) % M] = False trueA = trueA[:, keep] trueS = np.random.rand(N, T) < firerate / float(framerate) trueS[:, 0] = 0 for i in range(N // 2): trueS[i, :500 + i * T // N * 2 // 3] = 0 trueC = trueS.astype('float32') for i in range(N): # * (.9 + .2 * np.random.rand()))) gamma = np.exp(-1. / (tau * framerate)) for t in range(1, T): trueC[i, t] += gamma * trueC[i, t - 1] if fluctuating_bkgrd: K = np.array([[np.exp(-(i - j)**2 / 2. / fluctuating_bkgrd[0]**2) for i in range(T)] for j in range(T)]) ch = np.linalg.cholesky(K + 1e-10 * np.eye(T)) truef = 1e-2 * ch.dot(np.random.randn(T)).astype('float32') / bkgrd truef -= truef.mean() truef += 1 K = np.array([[np.exp(-(i - j)**2 / 2. / fluctuating_bkgrd[1]**2) for i in range(dims[0])] for j in range(dims[0])]) ch = np.linalg.cholesky(K + 1e-10 * np.eye(dims[0])) trueb = 3 * 1e-2 * \ np.outer( *ch.dot(np.random.randn(dims[0], 2)).T).ravel().astype('float32') trueb -= trueb.mean() trueb += 1 else: truef = np.ones(T, dtype='float32') trueb = np.ones(np.prod(dims), dtype='float32') trueb *= bkgrd Yr = np.outer(trueb, truef) + noise * np.random.randn( * (np.prod(dims), T)).astype('float32') + trueA.dot(trueC) if cmap: import matplotlib.pyplot as plt import caiman as cm Y = np.reshape(Yr, dims + (T,), order='F') Cn = cm.local_correlations(Y) plt.figure(figsize=(20, 3)) plt.plot(trueC.T) plt.figure(figsize=(20, 3)) plt.plot((trueA.T.dot(Yr - bkgrd) / np.sum(trueA**2, 0).reshape(-1, 1)).T) plt.figure(figsize=(12, 4)) plt.subplot(131) plt.scatter(*centers[keep].T[::-1], c='g') plt.scatter(*centers[~keep].T[::-1], c='r') plt.imshow(Y[:T // 10 * 10].reshape(dims + (T // 10, 10)).mean(-1).max(-1), cmap=cmap) plt.title('Max') plt.subplot(132) plt.scatter(*centers[keep].T[::-1], c='g') plt.scatter(*centers[~keep].T[::-1], c='r') plt.imshow(Y.mean(-1), cmap=cmap) plt.title('Mean') plt.subplot(133) plt.scatter(*centers[keep].T[::-1], c='g') plt.scatter(*centers[~keep].T[::-1], c='r') plt.imshow(Cn, cmap=cmap) plt.title('Correlation') plt.show() return Yr, trueC, trueS, trueA, trueb, truef, centers, dims
def initialize_movie_online(Y, K, gSig, rf, stride, base_name, p=1, merge_thresh=0.95, rval_thr_online=0.9, thresh_fitness_delta_online=-30, thresh_fitness_raw_online=-50, rval_thr_init=.5, thresh_fitness_delta_init=-20, thresh_fitness_raw_init=-20, rval_thr_refine=0.95, thresh_fitness_delta_refine=-100, thresh_fitness_raw_refine=-100, final_frate=10, Npeaks=10, single_thread=True, dview=None, n_processes=None): """ Initialize movie using CNMF on minibatch. See CNMF parameters """ _, d1, d2 = Y.shape dims = (d1, d2) Yr = Y.to_2D().T # merging threshold, max correlation allowed # order of the autoregressive system #T = Y.shape[0] base_name = base_name + '.mmap' fname_new = Y.save(base_name, order='C') #% Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') Cn2 = cm.local_correlations(Y) # pl.imshow(Cn2) #% #% RUN ALGORITHM ON PATCHES # pl.close('all') cnm_init = cm.source_extraction.cnmf.CNMF( n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, Ain=None, rf=rf, stride=stride, method_deconvolution='oasis', skip_refinement=False, normalize_init=False, options_local_NMF=None, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr_online, thresh_fitness_delta=thresh_fitness_delta_online, thresh_fitness_raw=thresh_fitness_raw_online, batch_update_suff_stat=True, max_comp_update_shape=5) cnm_init = cnm_init.fit(images) A_tot = cnm_init.A C_tot = cnm_init.C YrA_tot = cnm_init.YrA b_tot = cnm_init.b f_tot = cnm_init.f print(('Number of components:' + str(A_tot.shape[-1]))) #% traces = C_tot + YrA_tot # traces_a=traces-scipy.ndimage.percentile_filter(traces,8,size=[1,np.shape(traces)[-1]/5]) # traces_b=np.diff(traces,axis=1) fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = cm.components_evaluation.evaluate_components( Y, traces, A_tot, C_tot, b_tot, f_tot, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= rval_thr_init)[0] idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_init)[0] idx_components_delta = np.where( fitness_delta < thresh_fitness_delta_init)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components) print(('Keeping ' + str(len(idx_components)) + ' and discarding ' + str(len(idx_components_bad)))) A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] #% cnm_refine = cm.source_extraction.cnmf.CNMF( n_processes, method_init='greedy_roi', k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, rf=None, stride=None, p=p, dview=dview, Ain=A_tot, Cin=C_tot, f_in=f_tot, method_deconvolution='oasis', skip_refinement=True, normalize_init=False, options_local_NMF=None, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr_refine, thresh_fitness_delta=thresh_fitness_delta_refine, thresh_fitness_raw=thresh_fitness_raw_refine, batch_update_suff_stat=True, max_comp_update_shape=5) cnm_refine = cnm_refine.fit(images) #% A, C, b, f, YrA, sn = cnm_refine.A, cnm_refine.C, cnm_refine.b, cnm_refine.f, cnm_refine.YrA, cnm_refine.sn #% final_frate = 10 Npeaks = 10 traces = C + YrA fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \ cm.components_evaluation.evaluate_components(Y, traces, A, C, b, f, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= rval_thr_refine)[0] idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_refine)[0] idx_components_delta = np.where( fitness_delta < thresh_fitness_delta_refine)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components) print(' ***** ') print((len(traces))) print((len(idx_components))) #% cnm_refine.idx_components = idx_components cnm_refine.idx_components_bad = idx_components_bad cnm_refine.r_values = r_values cnm_refine.fitness_raw = fitness_raw cnm_refine.fitness_delta = fitness_delta cnm_refine.Cn2 = Cn2 #% # cnm_init.dview = None # save_object(cnm_init,fls[0][:-4]+ '_DS_' + str(ds)+ '_init.pkl') return cnm_refine, Cn2, fname_new
else: print('One file only, not saving!') fname_new = name_new[0] #%% # fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap' Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') #%% if np.min(images) < 0: raise Exception('Movie too negative, add_to_movie should be larger') if np.sum(np.isnan(images)) > 0: raise Exception('Movie contains nan! You did not remove enough borders') #%% Cn = cm.local_correlations(Y[:, :, :3000]) pl.imshow(Cn, cmap='gray') #%% if not is_patches: K = 35 # number of neurons expected per patch gSig = [7, 7] # expected half size of neurons merge_thresh = 0.8 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system cnm = cnmf.CNMF(n_processes, method_init=init_method, k=K, gSig=gSig, merge_thresh=merge_thresh, p=p, dview=dview,
name_new.sort() print(name_new) #%% fname_new = cm.save_memmap_join( name_new, base_name='Yr', n_chunks=12, dview=dview) #%% # fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap' Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') #%% if np.min(images) < 0: raise Exception('Movie too negative, add_to_movie should be larger') #%% Cn = cm.local_correlations(Y[:, :, :3000]) pl.imshow(Cn, cmap='gray') #%% # THIS METHOd CAN GIVE POSSIBLY INCONSISTENT RESULTS ON SOMAS WHEN NOT USED WITH PATCHES init_method = 'sparse_nmf' alpha_snmf = 10e3 # this controls sparsity! #%% rf = 30 # half-size of the patches in pixels. rf=25, patches are 50x50 stride = 10 # amounpl.it of overlap between the patches in pixels K = 10 # number of neurons expected per patch gSig = [6, 6] # expected half size of neurons merge_thresh = 0.8 # merging threshold, max correlation allowed p = 1 # order of the autoregressive system memory_fact = 1 # unitless number accounting how much memory should be used. You will need to try different values to see which one would work the default is OK for a 16 GB system
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set this to true to show movies and plots fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. To # close the video press q m_orig = cm.load_movie_chain(fname[:1]) downsample_ratio = 0.2 offset_mov = -np.min(m_orig[:100]) moviehandle = m_orig.resize(1, 1, downsample_ratio) if display_images: moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # maximum shift to be used for trimming against NaNs #%% compare with original movie moviehandle = cm.concatenate([ m_orig.resize(1, 1, downsample_ratio) + offset_mov, m_els.resize(1, 1, downsample_ratio) ], axis=2) display_images = False if display_images: moviehandle.play(fr=60, q_max=99.5, magnification=2, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) #%% PLOT COMPONENTS if display_images: plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) if display_images: view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=Cn) view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components_bad], cnm.C[idx_components_bad], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components_bad], img=Cn) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape( dims + (-1, ), order='F').transpose([2, 0, 1]) #%% play along side original data moviehandle = cm.concatenate([ m_els.resize(1, 1, downsample_ratio), denoised.resize(1, 1, downsample_ratio) ], axis=2) if display_images: moviehandle.play(fr=60, gain=15, magnification=2, offset=0) # press q to exit
gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview = dview, min_SNR=min_SNR,
def process_data(haussio_data, mask=None, p=2, nrois_init=400, roi_iceberg=0.9): if mask is not None: raise RuntimeError("mask not supported in cnmf.process_data") fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' shapefn = os.path.join(haussio_data.dirname_comp, haussio.THOR_RAW_FN[:-3] + "shape.npy") shape = np.load(shapefn) if len(shape) == 5: d1, d2 = shape[2], shape[3] fn_mmap = get_mmap_name('Yr', shape[2], shape[3], shape[0]) else: d1, d2 = shape[1], shape[2] fn_mmap = get_mmap_name('Yr', shape[1], shape[2], shape[0]) fn_mmap = os.path.join(haussio_data.dirname_comp, fn_mmap) print(fn_mmap, os.path.exists(fn_mmap), d1, d2) if not os.path.exists(fn_cnmf): # fn_raw = os.path.join(haussio_data.dirname_comp, haussio.THOR_RAW_FN) fn_sima = haussio_data.dirname_comp + '.sima' fnames = [ fn_sima, ] fnames.sort() print(fnames) fnames = fnames final_frate = 1.0 / haussio_data.dt downsample_factor = 1 # use .2 or .1 if file is large and you want a quick answer final_frate *= downsample_factor c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) idx_xy = None base_name = 'Yr' name_new = cm.save_memmap_each(fnames, dview=dview, base_name=base_name, resize_fact=(1, 1, downsample_factor), remove_init=0, idx_xy=idx_xy) name_new.sort() print(name_new) if len(name_new) > 1: fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=12, dview=dview) else: sys.stdout.write('One file only, not saving\n') fname_new = name_new[0] print("fname_new: " + fname_new) Yr, dims, T = cm.load_memmap(fname_new) Y = np.reshape(Yr, dims + (T, ), order='F') Cn = cm.local_correlations(Y) K = nrois_init # number of neurons expected per patch gSig = [15, 15] # expected half size of neurons merge_thresh = 0.8 # merging threshold, max correlation allowed p = 2 #order of the autoregressive system options = caiman_cnmf.utilities.CNMFSetParms(Y, NCPUS, p=p, gSig=gSig, K=K, ssub=2, tsub=2) Yr, sn, g, psx = caiman_cnmf.pre_processing.preprocess_data( Yr, dview=dview, **options['preprocess_params']) Atmp, Ctmp, b_in, f_in, center = caiman_cnmf.initialization.initialize_components( Y, **options['init_params']) Ain, Cin = Atmp, Ctmp A, b, Cin, f_in = caiman_cnmf.spatial.update_spatial_components( Yr, Cin, f_in, Ain, sn=sn, dview=dview, **options['spatial_params']) options['temporal_params'][ 'p'] = 0 # set this to zero for fast updating without deconvolution C, A, b, f, S, bl, c1, neurons_sn, g, YrA = caiman_cnmf.temporal.update_temporal_components( Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = caiman_cnmf.merging.merge_components( Yr, A, b, C, f, S, sn, options['temporal_params'], options['spatial_params'], dview=dview, bl=bl, c1=c1, sn=neurons_sn, g=g, thr=merge_thresh, mx=50, fast_merge=True) A2, b2, C2, f = caiman_cnmf.spatial.update_spatial_components( Yr, C_m, f, A_m, sn=sn, dview=dview, **options['spatial_params']) options['temporal_params'][ 'p'] = p # set it back to original value to perform full deconvolution C2, A2, b2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = caiman_cnmf.temporal.update_temporal_components( Yr, A2, b2, C2, f, dview=dview, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) tB = np.minimum(-2, np.floor(-5. / 30 * final_frate)) tA = np.maximum(5, np.ceil(25. / 30 * final_frate)) Npeaks = 10 traces = C2 + YrA fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \ evaluate_components( Y, traces, A2, C2, b2, f2, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= .6)[0] idx_components_raw = np.where(fitness_raw < -60)[0] idx_components_delta = np.where(fitness_delta < -20)[0] min_radius = gSig[0] - 2 masks_ws, idx_blobs, idx_non_blobs = extract_binary_masks_blob( A2.tocsc(), min_radius, dims, num_std_threshold=1, minCircularity=0.6, minInertiaRatio=0.2, minConvexity=.8) idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_blobs = np.intersect1d(idx_components, idx_blobs) idx_components_bad = np.setdiff1d(range(len(traces)), idx_components) A2 = A2.tocsc()[:, idx_components] C2 = C2[idx_components, :] YrA = YrA[idx_components, :] S2 = S2[idx_components, :] # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise", i.e. traces = C+YrA) # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] Yr, dims, T = cm.load_memmap(fn_mmap) dims = dims[1:] Y = np.reshape(Yr, dims + (T, ), order='F') proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) # t0 = time.time() # sys.stdout.write("Ordering components... ") # sys.stdout.flush() # A_or, C_or, srt = cse.order_components(A2, C2) # sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cm.stop_server() polygons = contour(A2, Y.shape[0], Y.shape[1], thr=roi_iceberg) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, zproj, S2, Y, YrA
base_name=None) #%% n_chunks = 6 # increase this number if you have memory issues at this point fname_new = cm.save_memmap_join(name_new, base_name='MOV_EXAMPLE_20160706154257__', n_chunks=6, dview=dview) #%% Yr, dims, T = cm.load_memmap( 'MOV_EXAMPLE_20160706154257___d1_150_d2_330_d3_1_order_C_frames_2300_.mmap' ) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') #%% Cn = cm.local_correlations(Y[:, :, :3000], swap_dim=True) pl.imshow(Cn, cmap='gray') #%% rf = 10 # half-size of the patches in pixels. rf=25, patches are 50x50 stride = 4 # amounpl.it of overlap between the patches in pixels K = 4 # number of neurons expected per patch gSig = [5, 5] # expected half size of neurons merge_thresh = 0.8 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system memory_fact = 1 # unitless number accounting how much memory should be used. You will need to try different values to see which one would work the default is OK for a 16 GB system save_results = False #%% RUN ALGORITHM ON PATCHES cnm = cnmf.CNMF(n_processes, k=K, gSig=gSig, merge_thresh=0.8,
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters for data and motion correction # dataset dependent parameters fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid motion correction in um # motion correction parameters pwrigid_motion_correct = True # flag to select rigid vs pw_rigid motion correction max_shifts = tuple([int(a/b) for a, b in zip(max_shift_um, dxy)]) # maximum allow rigid shift in pixels # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = False if display_images: m_orig = cm.load_movie_chain(fname) downsample_ratio = 0.2 moviehandle = m_orig.resize(1, 1, downsample_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname, min_mov, dview=dview, max_shifts=max_shifts, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, border_nan='copy', upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre if pwrigid_motion_correct: mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. else: mc.motion_correct_rigid(save_movie=True) m_els = cm.load(mc.fname_tot_rig) bord_px_els = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) fnames = mc.fname_tot_rig # name of the rigidly corrected file. # maximum shift to be used for trimming against NaNs #%% compare with original movie if display_images: downsample_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, downsample_ratio) - min_mov, m_els.resize(1, 1, downsample_ratio)], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' border_to_0 = bord_px_els # exclude borders due to motion correction # border_to_0 = 0 if mc.border_nan is 'copy' else bord_px_els # you can include boundaries if you used the 'copy' option in the motion # correction, although be careful abou the components near the boundaries fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' # parameters for component evaluation opts = params.CNMFParams(dims=dims, fr=fr, decay_time=decay_time, method_init=method_init, gSig=gSig, merge_thresh=merge_thresh, p=p, gnb=gnb, k=K, rf=rf, stride=stride_cnmf, rolling_sum=True) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.set('temporal', {'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier cnm.params.set('quality', {'fr': fr, 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr}) cnm.estimates.evaluate_components(images, cnm.params, dview=dview) #%% PLOT COMPONENTS cnm.estimates.plot_contours(img=Cn, idx=cnm.estimates.idx_components) #%% VIEW TRACES (accepted and rejected) if display_images: cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components) cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components_bad) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.dview = None cnm2 = deepcopy(cnm) cnm2.dview = dview cnm2.params.set('patch', {'rf': None}) cnm2.params.set('temporal', {'p': p}) cnm2 = cnm2.fit(images) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(Yr, img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=True) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
def main(): #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 15.49 # imaging rate in frames per second decay_time = 0.9 # length of a typical transient in seconds dxy = (1.4, 1.4) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='_memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 3 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 20 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 3 # number of components per patch gSig = [7, 7] # expected half size of neurons in pixels method_init = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization s_min = -20 # minimum signal amplitude needed in order for a transient to be considered as activity # parameters for component evaluation opts_dict = {'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub, 's_min': s_min} opts.change_params(params_dict=opts_dict) # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # Overall minimum signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.15 # neurons with cnn probability lower than this value are rejected min_size_neuro = 0.1*gSig[0]*np.pi**2 max_size_neuro = 2.5*gSig[0]*np.pi**2 cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest, 'min_size_neuro': min_size_neuro, 'max_size_neuro': max_size_neuro,}) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) #%% update object with selected components cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #cnm2.estimates.threshold_spatial_components(maxthr=0.85) #cnm2.estimates.remove_small_large_neurons(min_size_neuro=min_size_neuro, max_size_neuro=max_size_neuro) # %% Generate heat image of video and save data Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm2.estimates.Cn = Cn cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) #Create denoised video as matrix denoised = cm.movie(cnm2.estimates.A.dot(cnm2.estimates.C) + \ cnm2.estimates.b.dot(cnm2.estimates.f)).reshape(dims + (-1,), order='F').transpose([2, 0, 1]) #Normalizing denoised matrix to the [0,255] range prior to type conversion max_pixel = np.nanmax(denoised) norm_denoised = (denoised/max_pixel)*255 #Type conversion so imageio doesn't get mad norm_denoised = np.uint8(norm_denoised) #Saving a denoised avi of the analyzed video sname = Path(args.in_file).stem + '.avi' imageio.mimwrite(sname, norm_denoised, fps=15.49)
def initialize_movie(Y, K, gSig, rf, stride, base_name, p=1, merge_thresh=0.95, rval_thr_online=0.9, thresh_fitness_delta_online=-30, thresh_fitness_raw_online=-50, rval_thr_init=.5, thresh_fitness_delta_init=-20, thresh_fitness_raw_init=-20, rval_thr_refine=0.95, thresh_fitness_delta_refine=-100, thresh_fitness_raw_refine=-100, final_frate=10, Npeaks=10, single_thread=True, dview=None): _, d1, d2 = Y.shape dims = (d1, d2) Yr = Y.to_2D().T # merging threshold, max correlation allowed # order of the autoregressive system #T = Y.shape[0] base_name = base_name + '.mmap' fname_new = Y.save(base_name, order='C') #% Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') Cn2 = cm.local_correlations(Y) # pl.imshow(Cn2) #% #% RUN ALGORITHM ON PATCHES # pl.close('all') cnm_init = cnmf.CNMF(n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, Ain=None, rf=rf, stride=stride, method_deconvolution='oasis', skip_refinement=False, normalize_init=False, options_local_NMF=None, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr_online, thresh_fitness_delta=thresh_fitness_delta_online, thresh_fitness_raw=thresh_fitness_raw_online, batch_update_suff_stat=True, max_comp_update_shape=5) cnm_init = cnm_init.fit(images) A_tot = cnm_init.A C_tot = cnm_init.C YrA_tot = cnm_init.YrA b_tot = cnm_init.b f_tot = cnm_init.f print(('Number of components:' + str(A_tot.shape[-1]))) #% traces = C_tot + YrA_tot # traces_a=traces-scipy.ndimage.percentile_filter(traces,8,size=[1,np.shape(traces)[-1]/5]) # traces_b=np.diff(traces,axis=1) fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = evaluate_components( Y, traces, A_tot, C_tot, b_tot, f_tot, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= rval_thr_init)[0] idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_init)[0] idx_components_delta = np.where( fitness_delta < thresh_fitness_delta_init)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components) print(('Keeping ' + str(len(idx_components)) + ' and discarding ' + str(len(idx_components_bad)))) A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] #% cnm_refine = cnmf.CNMF(n_processes, method_init='greedy_roi', k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, rf=None, stride=None, p=p, dview=dview, Ain=A_tot, Cin=C_tot, f_in=f_tot, method_deconvolution='oasis', skip_refinement=True, normalize_init=False, options_local_NMF=None, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr_refine, thresh_fitness_delta=thresh_fitness_delta_refine, thresh_fitness_raw=thresh_fitness_raw_refine, batch_update_suff_stat=True, max_comp_update_shape=5) cnm_refine = cnm_refine.fit(images) #% A, C, b, f, YrA, sn = cnm_refine.A, cnm_refine.C, cnm_refine.b, cnm_refine.f, cnm_refine.YrA, cnm_refine.sn #% final_frate = 10 Npeaks = 10 traces = C + YrA fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \ evaluate_components(Y, traces, A, C, b, f, final_frate, remove_baseline=True, N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3) idx_components_r = np.where(r_values >= rval_thr_refine)[0] idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_refine)[0] idx_components_delta = np.where( fitness_delta < thresh_fitness_delta_refine)[0] idx_components = np.union1d(idx_components_r, idx_components_raw) idx_components = np.union1d(idx_components, idx_components_delta) idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components) print(' ***** ') print((len(traces))) print((len(idx_components))) #% cnm_refine.idx_components = idx_components cnm_refine.idx_components_bad = idx_components_bad cnm_refine.r_values = r_values cnm_refine.fitness_raw = fitness_raw cnm_refine.fitness_delta = fitness_delta cnm_refine.Cn2 = Cn2 #% # cnm_init.dview = None # save_object(cnm_init,fls[0][:-4]+ '_DS_' + str(ds)+ '_init.pkl') return cnm_refine, Cn2, fname_new
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # num processes n_proc = 12 # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_proc, single_thread=False) print('Parallel processing initialized.') print('Beginning motion correction') #%%% MOTION CORRECTION t_ms = time.time() # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative if not nomc: mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig,splits_rig=splits_rig,strides=strides, overlaps=overlaps, splits_els=splits_els,upsample_factor_grid=upsample_factor_grid,max_deviation_rigid=max_deviation_rigid,shifts_opencv=True, nonneg_movie=True, use_cuda=use_cuda) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),np.max(np.abs(mc.y_shifts_els)))).astype(np.int) t_mf = time.time() print('Motion correction complete in ', int(t_mf - t_ms),' seconds') # maximum shift to be used for trimming against NaNs #%% compare with original movie #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders bord_px_els = 5 # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_proc, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) print('Beginning initial CNMF fit') t1 = time.time() cnm = cnmf.CNMF(n_processes=n_proc, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) t2 = time.time() print('Initial CNMF fit complete in ', int(t2-t1), 'seconds.') Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier t3 = time.time() print('Estimating component quality') idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) t4 = time.time() print('Component quality estimation complete in ', int(t4-t3),' seconds') #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution print('Re-running CNMF to refine and deconvolve') t5 = time.time() A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=n_proc, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) t6 = time.time() print('CNMF re-run complete in ', int(t6-t5), ' seconds') #%% Extract DF/F values # F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, # quantileMin=8, frames_window=250) save_results = True if save_results: if os.path.isdir(infile[0]): outfile = os.path.join(infile[0],'caiman_output.npz') else: outfile = os.path.join(os.path.split(infile[0])[0],'caiman_output.npz') np.savez_compressed(outfile,Cn=Cn, A=cnm2.A.todense(), C=cnm2.C,b=cnm2.b, f=cnm2.f, YrA=cnm2.YrA, d1=d1, d2=d2,idx_components=idx_components, idx_components_bad=idx_components_bad) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview)
fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=12, dview=dview) else: fname_new = name_new[0] #%% from time import time # fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap' Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') #%% #if np.min(images)<0: # raise Exception('Movie too negative, add_to_movie should be larger') #%% if 0: Cn = cm.local_correlations(Y[:,:,::np.int(old_div(np.shape(images)[0],3000))]) pl.imshow(Cn) #%% #if not is_patches: # #%% # K = 35 # number of neurons expected per patch # gSig = [7, 7] # expected half size of neurons # merge_thresh = 0.8 # merging threshold, max correlation allowed # p = 2 # order of the autoregressive system # cnm = cnmf.CNMF(n_processes, method_init=init_method, k=K, gSig=gSig, merge_thresh=merge_thresh, # p=p, dview=dview, Ain=None,method_deconvolution='oasis') # cnm = cnm.fit(images) # ##%% #else:
m1.save('MOV_EXAMPLE_20160706154257.tif') #%% name_new = cm.save_memmap_each( ['MOV_EXAMPLE_20160706154257.tif'], dview=c[:], base_name=None) #%% n_chunks = 6 # increase this number if you have memory issues at this point fname_new = cm.save_memmap_join( name_new, base_name='MOV_EXAMPLE_20160706154257__', n_chunks=6, dview=dview) #%% Yr, dims, T = cm.load_memmap( 'MOV_EXAMPLE_20160706154257___d1_150_d2_330_d3_1_order_C_frames_2300_.mmap') d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') #%% Cn = cm.local_correlations(Y[:, :, :3000], swap_dim=True) pl.imshow(Cn, cmap='gray') #%% rf = 10 # half-size of the patches in pixels. rf=25, patches are 50x50 stride = 4 # amounpl.it of overlap between the patches in pixels K = 4 # number of neurons expected per patch gSig = [5, 5] # expected half size of neurons merge_thresh = 0.8 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system memory_fact = 1 # unitless number accounting how much memory should be used. You will need to try different values to see which one would work the default is OK for a 16 GB system save_results = False #%% RUN ALGORITHM ON PATCHES cnm = cnmf.CNMF(n_processes, k=K, gSig=gSig, merge_thresh=0.8, p=0, dview=dview, Ain=None, rf=rf, stride=stride, memory_fact=memory_fact, method_init='greedy_roi', alpha_snmf=10e2) cnm = cnm.fit(images)
plt.savefig(os.path.join(root, 'all_components.png')) plt.close() #%% serial evaluation root = r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch3\M32\20200318' #%% perform extraction motion_file, dview = pipe.motion_correction(root, opts, dview, remove_f_order=True, remove_c_order=True) root = r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch3\M32\20200322' mmap_file, images = pipe.load_mmap(root) cnm_params = cnm_params.change_params({'fnames': mmap_file[0]}) cnm = pipe.run_source_extraction(images, cnm_params, dview=dview) pipe.save_cnmf(cnm, path=os.path.join(root, 'cnm_pre_selection.hdf5'), verbose=False, overwrite=True) lcm = cm.local_correlations(images, swap_dim=False) lcm[np.isnan(lcm)] = 0 cnm.estimates.Cn = lcm pipe.save_cnmf(cnm, path=os.path.join(root, 'cnm_pre_selection.hdf5'), verbose=False, overwrite=True) cnm.estimates.plot_contours(img=cnm.estimates.Cn, display_numbers=False) plt.tight_layout() fig = plt.gcf() fig.set_size_inches((10, 10)) plt.savefig(os.path.join(root, 'pre_sel_components.png')) pipe.save_cnmf(cnm, path=os.path.join(root, 'cnm_pre_selection.hdf5'), verbose=False, overwrite=True) plt.close() #%% load pre-extracted data cnm = pipe.load_cnmf(root, 'cnm_pre_selection.hdf5') mmap_file, images = pipe.load_mmap(root)
#idx_xy=(idx_x,idx_y) add_to_movie=0 # of movie too negative need to add a baseline downsample_factor=1 # use .2 or .1 if file is large and you want a quick answer idx_xy=None base_name='Yr' name_new=cm.save_memmap_each(fnames, dview=dview,base_name=base_name, resize_fact=(1, 1, downsample_factor), remove_init=0,idx_xy=idx_xy,add_to_movie=add_to_movie ) name_new.sort() print(name_new) #%% fname_new=cm.save_memmap_join(name_new,base_name='Yr', n_chunks=12, dview=dview) #%% Yr,dims,T=cm.load_memmap(fname_new) Y=np.reshape(Yr,dims+(T,),order='F') #%% visualize correlation image Cn = cm.local_correlations(Y) pl.imshow(Cn,cmap='gray') #%% parameters of experiment K=30 # number of neurons expected per patch gSig=[7,7] # expected half size of neurons merge_thresh=0.8 # merging threshold, max correlation allowed p=2 #order of the autoregressive system options = cnmf.utilities.CNMFSetParms(Y,n_processes,p=p,gSig=gSig,K=K,ssub=2,tsub=2,nb=1) options['preprocess_params']['noise_method']='mean' #%% PREPROCESS DATA AND INITIALIZE COMPONENTS t1 = time() Yr,sn,g,psx = cm.source_extraction.cnmf.pre_processing.preprocess_data(Yr,dview=dview,**options['preprocess_params']) print((time() - t1)) #%% t1 = time() Atmp, Ctmp, b_in, f_in, center=cm.source_extraction.cnmf.initialization.initialize_components(Y, normalize=True, **options['init_params'])
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) root = '/Users/hheiser/Desktop/testing data/chronic_M2N3/0d_baseline/channel1' fnames = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\1\file_00003.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\2\file_00004.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\3\file_00005.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\4\file_00006.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\5\file_00007.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\6\file_00008.tif'] fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) #%% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=30, magnification=1, do_loop=False) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory #%% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) #%% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames[:3]) m_els = cm.load(mmap_file[:3]) ds_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie, m_els.resize(1, 1, ds_ratio)], axis=2) moviehandle.play(fr=15, q_max=99.5, magnification=2) # press q to exit #%% MEMORY MAPPING border_to_0 = 0 if mc.border_nan == 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries mmap_file = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\1\file_00001_els__d1_512_d2_512_d3_1_order_F_frames_1147_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\2\file_00002_els__d1_512_d2_512_d3_1_order_F_frames_2520_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\3\file_00003_els__d1_512_d2_512_d3_1_order_F_frames_3814_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\4\file_00004_els__d1_512_d2_512_d3_1_order_F_frames_5154_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\5\file_00005_els__d1_512_d2_512_d3_1_order_F_frames_2677_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\6\file_00006_els__d1_512_d2_512_d3_1_order_F_frames_3685_.mmap'] fname_new = r'E:\PhD\Data\DG\M14_20191014\N1\memmap__d1_512_d2_512_d3_1_order_C_frames_34939_.mmap' # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = {'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub} opts.change_params(params_dict=opts_dict) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) #opts.change_params({'p': 1,'rf':None, 'only_init':False}) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) #%% RUN CNMF SEEDED WITH MANUAL MASK # load mask mask = np.asarray(imageio.imread('/Users/hheiser/Desktop/testing data/file_00020_no_motion/avg_mask_fixed.png'), dtype=bool) # get component ROIs from the mask and plot them Ain, labels, mR = cm.base.rois.extract_binary_masks(mask) # plot original mask and extracted labels to check mask fig, ax = plt.subplots(1,2) ax[0].imshow(-mR,cmap='binary') ax[0].set_title('Original mask') ax[1].imshow(labels) ax[1].set_title('Extracted labelled ROIs') """" plt.figure() crd = cm.utils.visualization.plot_contours( Ain.astype('float32'), mR, thr=0.99, display_numbers=True) # todo check if this is important for the pipeline plt.title('Contour plots of detected ROIs in the structural channel') """ opts.change_params({'rf': None, 'only_init': False}) # run CNMF seeded with this mask cnm_corr = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain) cnm_corr_seed = cnm_corr_seed.fit(images) #cnm_seed = cnm_seed.fit_file(motion_correct=False) #%% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) #%% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.evaluate_components(images, params=cnm2.params, dview=dview) #%% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) #%% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components #### -> will delete rejected components! cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=1, bpx=border_to_0, include_bck=True) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% save results dirname = fnames[0][:-4] + "_results.hdf5" cnm2.estimates.Cn = Cn cnm2.save(dirname) #load results cnm2 = cnmf.load_CNMF(dirname) mov_name = fnames[0][:-4] + "_movie_restored_2_gain.tif" helper.save_movie(cnm2.estimates,images,mov_name,frame_range=range(200),include_bck=True)
from copy import deepcopy from scipy.special import log_ndtr import sys, os #******************************************************************************************************************************** #******************************************************************************************************************************** #******************************************************************************************************************************** #%% #fname = './example_movies/demoMovie.tif' fname = sys.argv[1] #'/mnt/244f644c-15b8-46be-8f8a-50b5b2d8c6e1/in_vivo/rafa/alejandro/G2M5/20170511/000/G2M5_C1V1_GCaMP6s_20170511_000.tif' print ("...running demo_OnACID on file: ", fname) Y = cm.load(fname).astype(np.float32) # Cn = cm.local_correlations(Y.transpose(1, 2, 0)) # used as a background image #Load parameters from .npz file if True: print (sys.argv[1][:-4]+"_runtime_params.npz") params = np.load(sys.argv[1][:-4]+"_runtime_params.npz") print (type(params['merge_thr'])) merge_thresh=np.float32(params['merge_thr']); print (merge_thresh) initbatch = np.int32(params['initibatch']); print (initbatch) patch_size = np.int32(params['patch_size']); print (patch_size) stride=np.int32(params['stride']); print (stride) K=np.int32(params['K']); print (K) frame_rate=np.float32(params['frame_rate']); print (frame_rate)
Created on Wed Sep 20 17:52:23 2017 @author: jfriedrich """ import numpy as np import pylab as pl import caiman as cm from caiman.source_extraction import cnmf as cnmf from caiman.utils.visualization import view_patches_bar, plot_contours from copy import deepcopy #%% fname = './example_movies/demoMovie.tif' Y = cm.load(fname).astype(np.float32) Cn = cm.local_correlations(Y.transpose(1, 2, 0)) T1 = Y.shape[0] merge_thresh = 0.8 # merging threshold, max correlation allowed p = 1 # order of the autoregressive system initbatch = 400 expected_comps = 50 rf = 16 stride = 3 K = 4 gSig = [6, 6] # expected half size of neurons rval_thr = 0.95 thresh_fitness_delta = -50 thresh_fitness_raw = -50 fname_new = Y[:initbatch].save('demo.mmap', order='C')
def main(): pass # For compatibility between running under Spyder and the CLI #%% """ General parameters """ play_movie = 1 plot_extras = 1 plot_extras_cell = 1 compute_mc_metrics = 1 #%% Select file(s) to be processed (download if not present) """ Load file """ #fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed #if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: # fnames = [download_demo(fnames[0])] #fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif'] #f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\' f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\' f_name = 'Ch1' f_ext = 'tif' fnames = [f_dir + f_name + '.' + f_ext] #fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5'] #%% First setup some parameters for data and motion correction """ Parameters """ # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 1 #0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels #max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] max_shifts = [6, 6] # start a new patch for pw-rigid motion correction every x pixels #strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) strides = [48, 48] # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q if play_movie: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # type "mc."and press TAB to see all interesting associated variables and self. outputs # interesting outputs # saved file is mc.fname_tot_els / mc.fname_tot_rig # mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch # mc.coord_shifts_els: coordinates associated to patches with shifts # mc.total_template_els: updated template for pw # mc.total_template_rig: updated template for rigid # mc.templates_rig: templates for each iteration in rig #%% # compute metrics for the results (TAKES TIME!!) if compute_mc_metrics: # not finished bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) final_size = np.subtract( mc.total_template_els.shape, 2 * bord_px_els) # remove pixels in the boundaries winsize = 100 swap_dim = False resize_fact_flow = .2 # downsample for computing ROF tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction( fnames[0], final_size[0], final_size[1], swap_dim, winsize=winsize, play_flow=False, resize_fact_flow=resize_fact_flow) plt.figure() plt.plot(correlations_orig) # %% compare with original movie if play_movie: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit del m_orig del m_els if plot_extras: # plot total template plt.figure() plt.imshow(mc.total_template_els) plt.title('Template after iteration') # plot x and y corrections plt.figure() plt.plot(mc.shifts_rig) plt.title('Rigid motion correction xy movement') plt.legend(['x shift', 'y shift']) plt.xlabel('frames') # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 2 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 2 # number of components per patch gSig = [15, 15] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 1 # spatial subsampling during initialization tsub = 1 # temporal subsampling during intialization # parameters for component evaluation opts_dict = { 'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub } opts.change_params(params_dict=opts_dict) # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) if plot_extras_cell: num_cell_plot = 51 plt.figure() plt.plot(cnm.estimates.C[num_cell_plot, :]) plt.title('Temporal component') plt.legend(['Cell ' + str(num_cell_plot)]) # plot component sptial profile A # first convert back to dense components plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape( list(dims)) plt.figure() plt.imshow(plot_spat_A) plt.title('Spatial component cell ' + str(num_cell_plot)) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') if plot_extras: plt.figure() plt.imshow(Cn) plt.title('Local correlations') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.90 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Component selection: min_SNR=' + str(min_SNR) + '; rval_thr=' + str(rval_thr) + '; cnn prob range=[' + str(cnn_lowest) + ' ' + str(cnn_thr) + ']') # %% VIEW TRACES (accepted and rejected) if plot_extras: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Accepted') cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) plt.suptitle('Rejected') #plt.figure(); #plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:]) # # # # #plt.figure(); #plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]); #plt.plot(); #plt.show(); # # #plt.figure(); #plt.plot(cnm2.estimates.detrend_df_f[1,:]) # these store the good and bad components, and next step sorts them # cnm2.estimates.idx_components # cnm2.estimates.idx_components_bad #%% update object with selected components #cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) plt.suptitle("Final results") #%% Save the mc data as in cmn struct as well ## #mc_out = dict( # pw_rigid = mc.pw_rigid, # fname = mc.fname, # mmap_file = mc.mmap_file, # total_template_els = mc.total_template_els, # total_template_rig = mc.total_template_rig, # border_nan = mc.border_nan, # border_to_0 = mc.border_to_0, # x_shifts_els = mc.x_shifts_els, # y_shifts_els = mc.y_shifts_els, # Cn = Cn # ) # # #deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out) #%% reconstruct denoised movie (press q to exit) if play_movie: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) save_results = True if save_results: cnm2.save(fnames[0][:-4] + '_results.hdf5')
def run_gui(path=None, data=None): """ Loads PCF or CNMF data object from a provided path and loads the CaImAn GUI for component inspection. This GUI was tweaked to not require any storage-intensive mmap files, but can therefore not show individual frames. :param path: optional str, directory of the PCF or CNMF object from which component data should be loaded. If None and data=None, a window prompt will open to select a directory where to look for a CNMF/PCF file. :param data: optional, data in form of an already loaded cnm object can be provided directly :return: """ try: cv2.setNumThreads(1) except: print('Open CV is naturally single threaded') try: if __IPYTHON__: # print(1) # this is used for debugging purposes only. allows to reload classes # when changed get_ipython().magic('load_ext autoreload') get_ipython().magic('autoreload 2') except NameError: print('Not launched under iPython') def make_color_img(img, gain=255, min_max=None, out_type=np.uint8): if min_max is None: min_ = img.min() max_ = img.max() else: min_, max_ = min_max img = (img - min_) / (max_ - min_) * gain img = img.astype(out_type) img = np.dstack([img] * 3) return img ### FIND DATA ### if data is None: # different conditions on file loading (not necessary if data was already provided) if path is None: # if no path has been given, open a window prompt to select a directory F = FileDialog() # load object saved by CNMF path = F.getExistingDirectory( caption='Select folder from which to load a PCF or CNMF file') try: # first try to get CNMF data from a PCF object (should be most up-to-date) cnm_obj = pipe.load_pcf(path).cnmf except FileNotFoundError: try: cnm_obj = pipe.load_cnmf(path) except FileNotFoundError: raise FileNotFoundError( f'Could not find data to load in {path}!') else: cnm_obj = data # movie NOT NEEDED IN VERSION WITHOUT MMAP FILE # if not os.path.exists(cnm_obj.mmap_file): # M = FileDialog() # cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file', filter='*.mmap')[0] # # if fpath[-3:] == 'nwb': # mov = cm.load(cnm_obj.mmap_file, var_name_hdf5='acquisition/TwoPhotonSeries') # else: # mov = cm.load(cnm_obj.mmap_file) estimates = cnm_obj.estimates params_obj = cnm_obj.params # min_mov = np.min(mov) # max_mov = np.max(mov) if not hasattr(estimates, 'Cn'): if not os.path.exists(cnm_obj.mmap_file): M = FileDialog() cnm_obj.mmap_file = M.getOpenFileName( caption='Load memory mapped file', filter='*.mmap')[0] mov = cm.load(cnm_obj.mmap_file) estimates.Cn = cm.local_correlations(mov, swap_dim=False) Cn = estimates.Cn # min_mov_denoise = np.min(estimates.A)*estimates.C.min() # max_mov_denoise = np.max(estimates.A)*estimates.C.max() background_num = -1 neuron_selected = False nr_index = 0 min_background = np.min(estimates.b, axis=0) * np.min(estimates.f, axis=1) max_background = np.max(estimates.b, axis=0) * np.max(estimates.f, axis=1) if not hasattr(estimates, 'accepted_list'): # if estimates.discarded_components.A.shape[-1] > 0: # estimates.restore_discarded_components() estimates.accepted_list = np.array([], dtype=np.int) estimates.rejected_list = np.array([], dtype=np.int) estimates.img_components = estimates.A.toarray().reshape( (estimates.dims[0], estimates.dims[1], -1), order='F').transpose([2, 0, 1]) estimates.cms = np.array([ scipy.ndimage.measurements.center_of_mass(comp) for comp in estimates.img_components ]) estimates.idx_components = np.arange(estimates.nr) estimates.idx_components_bad = np.array([]) estimates.background_image = make_color_img(estimates.Cn) # Generate image data estimates.img_components /= estimates.img_components.max( axis=(1, 2))[:, None, None] estimates.img_components *= 255 estimates.img_components = estimates.img_components.astype(np.uint8) def draw_contours_overall(md): if md is "reset": draw_contours() elif md is "neurons": if neuron_selected is True: #if a specific neuron has been selected, only one contour should be changed while thrshcomp_line is changing if nr_index is 0: #if user does not start to move through the frames draw_contours_update(estimates.background_image, img) draw_contours_update(comp2_scaled, img2) else: # NEVER CALLED IN THIS VERSION WITHOUT MMAP SINCE NR_INDEX NEVER CHANGES (NO NR_VLINE) draw_contours_update(raw_mov_scaled, img) draw_contours_update(frame_denoise_scaled, img2) else: #if no specific neuron has been selected, all the contours are changing draw_contours() else: #md is "background": return def draw_contours(): global thrshcomp_line, estimates, img bkgr_contours = estimates.background_image.copy() if len(estimates.idx_components) > 0: contours = [ cv2.findContours( cv2.threshold(img, np.int(thrshcomp_line.value()), 255, 0)[1], cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)[0] for img in estimates.img_components[estimates.idx_components] ] SNRs = np.array(estimates.r_values) iidd = np.array(estimates.idx_components) idx1 = np.where(SNRs[iidd] < 0.1)[0] idx2 = np.where((SNRs[iidd] >= 0.1) & (SNRs[iidd] < 0.25))[0] idx3 = np.where((SNRs[iidd] >= 0.25) & (SNRs[iidd] < 0.5))[0] idx4 = np.where((SNRs[iidd] >= 0.5) & (SNRs[iidd] < 0.75))[0] idx5 = np.where((SNRs[iidd] >= 0.75) & (SNRs[iidd] < 0.9))[0] idx6 = np.where(SNRs[iidd] >= 0.9)[0] cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx1], []), -1, (255, 0, 0), 1) cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx2], []), -1, (0, 255, 0), 1) cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx3], []), -1, (0, 0, 255), 1) cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx4], []), -1, (255, 255, 0), 1) cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx5], []), -1, (255, 0, 255), 1) cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx6], []), -1, (0, 255, 255), 1) img.setImage(bkgr_contours, autoLevels=False) # pg.setConfigOptions(imageAxisOrder='row-major') def draw_contours_update(cf, im): global thrshcomp_line, estimates curFrame = cf.copy() if len(estimates.idx_components) > 0: contours = [ cv2.findContours( cv2.threshold(img, np.int(thrshcomp_line.value()), 255, 0)[1], cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)[0] for img in estimates.img_components[estimates.idx_components] ] SNRs = np.array(estimates.r_values) iidd = np.array(estimates.idx_components) idx1 = np.where(SNRs[iidd] < 0.1)[0] idx2 = np.where((SNRs[iidd] >= 0.1) & (SNRs[iidd] < 0.25))[0] idx3 = np.where((SNRs[iidd] >= 0.25) & (SNRs[iidd] < 0.5))[0] idx4 = np.where((SNRs[iidd] >= 0.5) & (SNRs[iidd] < 0.75))[0] idx5 = np.where((SNRs[iidd] >= 0.75) & (SNRs[iidd] < 0.9))[0] idx6 = np.where(SNRs[iidd] >= 0.9)[0] if min_dist_comp in idx1: cv2.drawContours(curFrame, contours[min_dist_comp], -1, (255, 0, 0), 1) if min_dist_comp in idx2: cv2.drawContours(curFrame, contours[min_dist_comp], -1, (0, 255, 0), 1) if min_dist_comp in idx3: cv2.drawContours(curFrame, contours[min_dist_comp], -1, (0, 0, 255), 1) if min_dist_comp in idx4: cv2.drawContours(curFrame, contours[min_dist_comp], -1, (255, 255, 0), 1) if min_dist_comp in idx5: cv2.drawContours(curFrame, contours[min_dist_comp], -1, (255, 0, 255), 1) if min_dist_comp in idx6: cv2.drawContours(curFrame, contours[min_dist_comp], -1, (0, 255, 255), 1) im.setImage(curFrame, autoLevels=False) #%% START BUILDING THE APPLICATION WINDOW # Always start by initializing Qt (only once per application) app = QtGui.QApplication([]) # Define a top-level widget to hold everything w = QtGui.QWidget() # Create some widgets to be placed inside btn = QtGui.QPushButton('press me') text = QtGui.QLineEdit('enter text') # Histogram controller (win) win = pg.GraphicsLayoutWidget() win.setMaximumWidth(300) win.setMinimumWidth(200) hist = pg.HistogramLUTItem() # Contrast/color control win.addItem(hist) # Plotting windows p1 = pg.PlotWidget( ) # raw movie window (top-mid), all contours are drawn here p2 = pg.PlotWidget( ) # trace window (bottom-mid), calcium trace of the selected component p3 = pg.PlotWidget( ) # denoised movie window (top-right), only selected contour is drawn here # parameter table for online evaluation and mode change t = ParameterTree() # parameter table for neuron selection/sorting t_action = ParameterTree() action_layout = QtGui.QGridLayout() ## Create a grid layout to manage the widgets size and position layout = QtGui.QGridLayout() w.setLayout(layout) # A plot area (ViewBox + axes) for displaying the image #p1 = win.addPlot(title="Image here") # Item for displaying image data img = pg.ImageItem() p1.addItem(img) img2 = pg.ImageItem() p3.addItem(img2) hist.setImageItem(img) # Draggable line for setting isocurve level thrshcomp_line = pg.InfiniteLine(angle=0, movable=True, pen='g') hist.vb.addItem(thrshcomp_line) hist.vb.setMouseEnabled(y=False) # makes user interaction a little easier thrshcomp_line.setValue(100) thrshcomp_line.setZValue(1000) # bring iso line above contrast controls ## Add widgets to the layout in their proper positions layout.addWidget(win, 1, 0) # histogram layout.addWidget(p3, 0, 2) # denoised movie layout.addWidget(t, 0, 0) # upper-right table layout.addWidget(t_action, 1, 2) # bottom-right table layout.addWidget(p1, 0, 1) # raw movie layout.addWidget(p2, 1, 1) # calcium trace window #enable only horizontal zoom for the traces component p2.setMouseEnabled(x=True, y=False) ## Display the widget as a new window w.show() ## Start the Qt event loop app.exec_() draw_contours() hist.setLevels(estimates.background_image.min(), estimates.background_image.max()) # Another plot area for displaying ROI data #win.nextRow() #p2 = win.addPlot(colspan=2) p2.setMaximumHeight(250) #win.resize(800, 800) #win.show() # set position and scale of image img.scale(1, 1) # img.translate(-50, 0) # zoom to fit image p1.autoRange() mode = "reset" p2.setTitle("mode: %s" % (mode)) thrshcomp_line.sigDragged.connect(lambda: draw_contours_overall(mode)) def imageHoverEvent(event): #Show the position, pixel, and value under the mouse cursor. global x, y, i, j, val pos = event.pos() i, j = pos.y(), pos.x() i = int(np.clip(i, 0, estimates.background_image.shape[0] - 1)) j = int(np.clip(j, 0, estimates.background_image.shape[1] - 1)) val = estimates.background_image[i, j, 0] ppos = img.mapToParent(pos) x, y = ppos.x(), ppos.y() # Monkey-patch the image to use our custom hover function. # This is generally discouraged (you should subclass ImageItem instead), # but it works for a very simple use like this. img.hoverEvent = imageHoverEvent def mouseClickEvent(event): global mode global x, y, i, j, val pos = img.mapFromScene(event.pos()) x = int(pos.x()) y = int(pos.y()) if x < 0 or x > mov.shape[1] or y < 0 or y > mov.shape[2]: # if the user click outside of the movie, do nothing and jump out of the function return i, j = pos.y(), pos.x() i = int(np.clip(i, 0, estimates.background_image.shape[0] - 1)) j = int(np.clip(j, 0, estimates.background_image.shape[1] - 1)) val = estimates.background_image[i, j, 0] if mode is "neurons": show_neurons_clicked() p1.mousePressEvent = mouseClickEvent #A general rule in Qt is that if you override one mouse event handler, you must override all of them. def release(event): pass p1.mouseReleaseEvent = release def move(event): pass p1.mouseMoveEvent = move ## PARAMS params = [{ 'name': 'min_cnn_thr', 'type': 'float', 'value': 0.99, 'limits': (0, 1), 'step': 0.01 }, { 'name': 'cnn_lowest', 'type': 'float', 'value': 0.1, 'limits': (0, 1), 'step': 0.01 }, { 'name': 'rval_thr', 'type': 'float', 'value': 0.85, 'limits': (-1, 1), 'step': 0.01 }, { 'name': 'rval_lowest', 'type': 'float', 'value': -1, 'limits': (-1, 1), 'step': 0.01 }, { 'name': 'min_SNR', 'type': 'float', 'value': 2, 'limits': (0, 20), 'step': 0.1 }, { 'name': 'SNR_lowest', 'type': 'float', 'value': 0, 'limits': (0, 20), 'step': 0.1 }, { 'name': 'RESET', 'type': 'action' }, { 'name': 'SHOW BACKGROUND', 'type': 'action' }, { 'name': 'SHOW NEURONS', 'type': 'action' }] ## Create tree of Parameter objects pars = Parameter.create(name='params', type='group', children=params) params_action = [{ 'name': 'Filter components', 'type': 'bool', 'value': True, 'tip': "Filter components" }, { 'name': 'View components', 'type': 'list', 'values': ['All', 'Accepted', 'Rejected', 'Unassigned'], 'value': 'All' }, { 'name': 'ADD GROUP', 'type': 'action' }, { 'name': 'REMOVE GROUP', 'type': 'action' }, { 'name': 'ADD SINGLE', 'type': 'action' }, { 'name': 'REMOVE SINGLE', 'type': 'action' }, { 'name': 'SAVE OBJECT', 'type': 'action' }] pars_action = Parameter.create(name='params_action', type='group', children=params_action) t_action.setParameters(pars_action, showTop=False) t_action.setWindowTitle('Parameter Action') def reset_button(): global mode mode = "reset" p2.setTitle("mode: %s" % (mode)) #clear the upper right image zeros = np.asarray([[0] * 80 for _ in range(60)]) img2.setImage(make_color_img(zeros), autoLevels=False) draw_contours() pars.param('RESET').sigActivated.connect(reset_button) def show_background_button(): global bg_vline, min_background, max_background, background_num global mode, background_first_frame_scaled #clear thhe upper right image zeros = np.asarray([[0] * 80 for _ in range(60)]) img2.setImage(make_color_img(zeros), autoLevels=False) background_num = (background_num + 1) % estimates.f.shape[0] mode = "background" p2.setTitle("mode: %s %d" % (mode, background_num)) # display the first frame of the background background_first_frame = estimates.b[:, background_num].reshape( estimates.dims, order='F') min_background_first_frame = np.min(background_first_frame) max_background_first_frame = np.max(background_first_frame) background_first_frame_scaled = make_color_img( background_first_frame, min_max=(min_background_first_frame, max_background_first_frame)) img.setImage(background_first_frame_scaled, autoLevels=False) # draw the trace and the infinite line trace_background = estimates.f[background_num] p2.plot(trace_background, clear=True) bg_vline = pg.InfiniteLine(angle=90, movable=True) p2.addItem(bg_vline, ignoreBounds=True) bg_vline.setValue(0) bg_vline.sigPositionChanged.connect(show_background_update) def show_background_update(): global bg_index, min_background, max_background, background_scaled bg_index = int(bg_vline.value()) if bg_index > -1 and bg_index < estimates.f.shape[-1]: # upper left component scrolls through the frames of the background background = estimates.b[:, background_num].dot( estimates.f[background_num, bg_index]).reshape(estimates.dims, order='F') background_scaled = make_color_img( background, min_max=(min_background[background_num], max_background[background_num])) img.setImage(background_scaled, autoLevels=False) pars.param('SHOW BACKGROUND').sigActivated.connect(show_background_button) def show_neurons_button(): global mode, neuron_selected mode = "neurons" neuron_selected = False p2.setTitle("mode: %s" % (mode)) #clear the upper right image zeros = np.asarray([[0] * 80 for _ in range(60)]) img2.setImage(make_color_img(zeros), autoLevels=False) def show_neurons_clicked(): global nr_index global x, y, i, j, val, min_dist_comp, contour_single, neuron_selected, comp2_scaled neuron_selected = True distances = np.sum( ((x, y) - estimates.cms[estimates.idx_components])**2, axis=1)**0.5 min_dist_comp = np.argmin(distances) contour_all = [ cv2.threshold(img, np.int(thrshcomp_line.value()), 255, 0)[1] for img in estimates.img_components[estimates.idx_components] ] contour_single = contour_all[min_dist_comp] # draw the traces (lower left component) estimates.components_to_plot = estimates.idx_components[min_dist_comp] p2.plot(estimates.C[estimates.components_to_plot] + estimates.YrA[estimates.components_to_plot], clear=True) # plot img (upper left component) img.setImage(estimates.background_image, autoLevels=False) draw_contours_update(estimates.background_image, img) # plot img2 (upper right component) comp2 = np.multiply(estimates.Cn, contour_single > 0) comp2_scaled = make_color_img(comp2, min_max=(np.min(comp2), np.max(comp2))) img2.setImage(comp2_scaled, autoLevels=False) draw_contours_update(comp2_scaled, img2) # set title for the upper two components p3.setTitle("pos: (%0.1f, %0.1f) component: %d value: %g dist:%f" % (x, y, estimates.components_to_plot, val, distances[min_dist_comp])) p1.setTitle("pos: (%0.1f, %0.1f) component: %d value: %g dist:%f" % (x, y, estimates.components_to_plot, val, distances[min_dist_comp])) # draw the infinite line (INACTIVE IN THIS VERSION WITHOUT MMAP FILES) # nr_vline = pg.InfiniteLine(angle=90, movable=True) # p2.addItem(nr_vline, ignoreBounds=True) # nr_vline.setValue(0) # nr_vline.sigPositionChanged.connect(show_neurons_update) nr_index = 0 def show_neurons_update(): # NOT CALLED IN THIS VERSION global nr_index, frame_denoise_scaled, estimates, raw_mov_scaled global min_mov, max_mov, min_mov_denoise, max_mov_denoise if neuron_selected is False: return nr_index = int(nr_vline.value()) if nr_index > 0 and nr_index < mov[:, 0, 0].shape[0]: # upper left component scrolls through the raw movie raw_mov = mov[nr_index, :, :] raw_mov_scaled = make_color_img(raw_mov, min_max=(min_mov, max_mov)) img.setImage(raw_mov_scaled, autoLevels=False) draw_contours_update(raw_mov_scaled, img) # upper right component scrolls through the denoised movie frame_denoise = estimates.A[:, estimates.idx_components].dot( estimates.C[estimates.idx_components, nr_index]).reshape(estimates.dims, order='F') frame_denoise_scaled = make_color_img(frame_denoise, min_max=(min_mov_denoise, max_mov_denoise)) img2.setImage(frame_denoise_scaled, autoLevels=False) draw_contours_update(frame_denoise_scaled, img2) pars.param('SHOW NEURONS').sigActivated.connect(show_neurons_button) def add_group(): estimates.accepted_list = np.union1d(estimates.accepted_list, estimates.idx_components) estimates.rejected_list = np.setdiff1d(estimates.rejected_list, estimates.idx_components) change(None, None) pars_action.param('ADD GROUP').sigActivated.connect(add_group) def remove_group(): estimates.rejected_list = np.union1d(estimates.rejected_list, estimates.idx_components) estimates.accepted_list = np.setdiff1d(estimates.accepted_list, estimates.idx_components) change(None, None) pars_action.param('REMOVE GROUP').sigActivated.connect(remove_group) def add_single(): estimates.accepted_list = np.union1d(estimates.accepted_list, estimates.components_to_plot) estimates.rejected_list = np.setdiff1d(estimates.rejected_list, estimates.components_to_plot) change(None, None) pars_action.param('ADD SINGLE').sigActivated.connect(add_single) def remove_single(): estimates.rejected_list = np.union1d(estimates.rejected_list, estimates.components_to_plot) estimates.accepted_list = np.setdiff1d(estimates.accepted_list, estimates.components_to_plot) change(None, None) pars_action.param('REMOVE SINGLE').sigActivated.connect(remove_single) def save_object(): print('Saving') ffll = F.getSaveFileName(filter='*.hdf5') print(ffll[0]) cnm_obj.estimates = estimates cnm_obj.save(ffll[0]) pars_action.param('SAVE OBJECT').sigActivated.connect(save_object) def action_pars_activated(param, changes): change(None, None) pars_action.sigTreeStateChanged.connect(action_pars_activated) ## If anything changes in the tree, print a message def change(param, changes): global estimates, pars, pars_action set_par = pars.getValues() if pars_action.param('Filter components').value(): for keyy in set_par.keys(): params_obj.quality.update({keyy: set_par[keyy][0]}) else: params_obj.quality.update({ 'cnn_lowest': .1, 'min_cnn_thr': 0.99, 'rval_thr': 0.85, 'rval_lowest': -1, 'min_SNR': 2, 'SNR_lowest': 0 }) estimates.filter_components( mov, params_obj, dview=None, select_mode=pars_action.param('View components').value()) if mode is "background": return else: draw_contours() pars.sigTreeStateChanged.connect(change) change(None, None) # set params to default t.setParameters(pars, showTop=False) t.setWindowTitle('Parameter Quality') ## END PARAMS ## Display the widget as a new window w.show() ## Start the Qt event loop app.exit(app.exec_())
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a / b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = False if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = { 'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub } opts.change_params(params_dict=opts_dict) # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
def test_general(): """ General Test of pipeline with comparison against ground truth A shorter version than the demo pipeline that calls comparison for the real test work Raises: --------- params_movie params_cnmf rig correction cnmf on patch cnmf full frame not able to read the file no groundtruth """ #\bug #\warning global params_movie global params_diplay fname = params_movie['fname'] niter_rig = params_movie['niter_rig'] max_shifts = params_movie['max_shifts'] splits_rig = params_movie['splits_rig'] num_splits_to_process_rig = params_movie['num_splits_to_process_rig'] cwd = os.getcwd() fname = download_demo(fname[0]) m_orig = cm.load(fname) min_mov = m_orig[:400].min() comp = comparison.Comparison() comp.dims = np.shape(m_orig)[1:] ################ RIG CORRECTION ################# t1 = time.time() mc = MotionCorrect(fname, min_mov, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, num_splits_to_process_rig=num_splits_to_process_rig, shifts_opencv=True, nonneg_movie=True) mc.motion_correct_rigid(save_movie=True) m_rig = cm.load(mc.fname_tot_rig) bord_px_rig = np.ceil(np.max(mc.shifts_rig)).astype(np.int) comp.comparison['rig_shifts']['timer'] = time.time() - t1 comp.comparison['rig_shifts']['ourdata'] = mc.shifts_rig ########################################### if 'max_shifts' not in params_movie: fnames = params_movie['fname'] border_to_0 = 0 else: # elif not params_movie.has_key('overlaps'): fnames = mc.fname_tot_rig border_to_0 = bord_px_rig m_els = m_rig idx_xy = None add_to_movie = -np.nanmin(m_els) + 1 # movie must be positive remove_init = 0 downsample_factor = 1 base_name = fname[0].split('/')[-1][:-4] name_new = cm.save_memmap_each(fnames, base_name=base_name, resize_fact=( 1, 1, downsample_factor), remove_init=remove_init, idx_xy=idx_xy, add_to_movie=add_to_movie, border_to_0=border_to_0) name_new.sort() if len(name_new) > 1: fname_new = cm.save_memmap_join( name_new, base_name='Yr', n_chunks=params_movie['n_chunks'], dview=None) else: print('One file only, not saving!') fname_new = name_new[0] Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') if np.min(images) < 0: # TODO: should do this in an automatic fashion with a while loop at the 367 line raise Exception('Movie too negative, add_to_movie should be larger') if np.sum(np.isnan(images)) > 0: # TODO: same here raise Exception( 'Movie contains nan! You did not remove enough borders') Cn = cm.local_correlations(Y) Cn[np.isnan(Cn)] = 0 p = params_movie['p'] merge_thresh = params_movie['merge_thresh'] rf = params_movie['rf'] stride_cnmf = params_movie['stride_cnmf'] K = params_movie['K'] init_method = params_movie['init_method'] gSig = params_movie['gSig'] alpha_snmf = params_movie['alpha_snmf'] if params_movie['is_dendrites'] == True: if params_movie['init_method'] is not 'sparse_nmf': raise Exception('dendritic requires sparse_nmf') if params_movie['alpha_snmf'] is None: raise Exception('need to set a value for alpha_snmf') ################ CNMF PART PATCH ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], dview=None, rf=rf, stride=stride_cnmf, memory_fact=params_movie['memory_fact'], method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=params_movie[ 'only_init_patch'], gnb=params_movie['gnb'], method_deconvolution='oasis') comp.cnmpatch = copy.copy(cnm) cnm = cnm.fit(images) A_tot = cnm.A C_tot = cnm.C YrA_tot = cnm.YrA b_tot = cnm.b f_tot = cnm.f # DISCARDING print(('Number of components:' + str(A_tot.shape[-1]))) final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_patch'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_patch'] fitness_delta_min = params_movie['fitness_delta_min_patch'] Npeaks = params_movie['Npeaks'] traces = C_tot + YrA_tot idx_components, idx_components_bad = estimate_components_quality( traces, Y, A_tot, C_tot, b_tot, f_tot, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min) ####### A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1 comp.comparison['cnmf_on_patch']['ourdata'] = [A_tot.copy(), C_tot.copy()] #################### ######################## ################ CNMF PART FULL ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, p=p, Ain=A_tot, Cin=C_tot, f_in=f_tot, rf=None, stride=None, method_deconvolution='oasis') cnm = cnm.fit(images) # DISCARDING A, C, b, f, YrA, sn = cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, cnm.sn final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_full'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_full'] fitness_delta_min = params_movie['fitness_delta_min_full'] Npeaks = params_movie['Npeaks'] traces = C + YrA idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality( traces, Y, A, C, b, f, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True) ########## A_tot_full = A_tot.tocsc()[:, idx_components] C_tot_full = C_tot[idx_components] comp.comparison['cnmf_full_frame']['timer'] = time.time() - t1 comp.comparison['cnmf_full_frame']['ourdata'] = [ A_tot_full.copy(), C_tot_full.copy()] #################### ######################## comp.save_with_compare(istruth=False, params=params_movie, Cn=Cn) log_files = glob.glob('*_LOG_*') try: for log_file in log_files: os.remove(log_file) except: print('Cannot remove log files') ############ assertions ################## pb = False if (comp.information['differences']['params_movie']): print("you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)") pb = True if (comp.information['differences']['params_cnm']): print("you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)") pb = True if (comp.information['diff']['rig']['isdifferent']): print("the rigid shifts are different from the groundtruth ") pb = True if (comp.information['diff']['cnmpatch']['isdifferent']): print("the cnmf on patch produces different results than the groundtruth ") pb = True if (comp.information['diff']['cnmfull']['isdifferent']): print("the cnmf full frame produces different results than the groundtruth ") pb = True assert (not pb)
def test_general(): """ General Test of pipeline with comparison against ground truth A shorter version than the demo pipeline that calls comparison for the real test work Raises: --------- params_movie params_cnmf rig correction cnmf on patch cnmf full frame not able to read the file no groundtruth """ #\bug #\warning global params_movie global params_diplay fname = params_movie['fname'] niter_rig = params_movie['niter_rig'] max_shifts = params_movie['max_shifts'] splits_rig = params_movie['splits_rig'] num_splits_to_process_rig = params_movie['num_splits_to_process_rig'] cwd = os.getcwd() fname = download_demo(fname[0]) m_orig = cm.load(fname) min_mov = m_orig[:400].min() comp = comparison.Comparison() comp.dims = np.shape(m_orig)[1:] ################ RIG CORRECTION ################# t1 = time.time() mc = MotionCorrect(fname, min_mov, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, num_splits_to_process_rig=num_splits_to_process_rig, shifts_opencv=True, nonneg_movie=True) mc.motion_correct_rigid(save_movie=True) m_rig = cm.load(mc.fname_tot_rig) bord_px_rig = np.ceil(np.max(mc.shifts_rig)).astype(np.int) comp.comparison['rig_shifts']['timer'] = time.time() - t1 comp.comparison['rig_shifts']['ourdata'] = mc.shifts_rig ########################################### if 'max_shifts' not in params_movie: fnames = params_movie['fname'] border_to_0 = 0 else: # elif not params_movie.has_key('overlaps'): fnames = mc.fname_tot_rig border_to_0 = bord_px_rig m_els = m_rig idx_xy = None add_to_movie = -np.nanmin(m_els) + 1 # movie must be positive remove_init = 0 downsample_factor = 1 base_name = fname[0].split('/')[-1][:-4] name_new = cm.save_memmap_each(fnames, base_name=base_name, resize_fact=(1, 1, downsample_factor), remove_init=remove_init, idx_xy=idx_xy, add_to_movie=add_to_movie, border_to_0=border_to_0) name_new.sort() if len(name_new) > 1: fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=params_movie['n_chunks'], dview=None) else: logging.warning('One file only, not saving!') fname_new = name_new[0] Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') if np.min(images) < 0: # TODO: should do this in an automatic fashion with a while loop at the 367 line raise Exception('Movie too negative, add_to_movie should be larger') if np.sum(np.isnan(images)) > 0: # TODO: same here raise Exception( 'Movie contains nan! You did not remove enough borders') Cn = cm.local_correlations(Y) Cn[np.isnan(Cn)] = 0 p = params_movie['p'] merge_thresh = params_movie['merge_thresh'] rf = params_movie['rf'] stride_cnmf = params_movie['stride_cnmf'] K = params_movie['K'] init_method = params_movie['init_method'] gSig = params_movie['gSig'] alpha_snmf = params_movie['alpha_snmf'] if params_movie['is_dendrites'] == True: if params_movie['init_method'] is not 'sparse_nmf': raise Exception('dendritic requires sparse_nmf') if params_movie['alpha_snmf'] is None: raise Exception('need to set a value for alpha_snmf') ################ CNMF PART PATCH ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=params_movie['merge_thresh'], p=params_movie['p'], dview=None, rf=rf, stride=stride_cnmf, memory_fact=params_movie['memory_fact'], method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=params_movie['only_init_patch'], gnb=params_movie['gnb'], method_deconvolution='oasis') comp.cnmpatch = copy.copy(cnm) comp.cnmpatch.estimates = None cnm = cnm.fit(images) A_tot = cnm.estimates.A C_tot = cnm.estimates.C YrA_tot = cnm.estimates.YrA b_tot = cnm.estimates.b f_tot = cnm.estimates.f # DISCARDING logging.info(('Number of components:' + str(A_tot.shape[-1]))) final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_patch'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_patch'] fitness_delta_min = params_movie['fitness_delta_min_patch'] Npeaks = params_movie['Npeaks'] traces = C_tot + YrA_tot idx_components, idx_components_bad = estimate_components_quality( traces, Y, A_tot, C_tot, b_tot, f_tot, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min) ####### A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1 comp.comparison['cnmf_on_patch']['ourdata'] = [A_tot.copy(), C_tot.copy()] #################### ######################## ################ CNMF PART FULL ################# t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, p=p, Ain=A_tot, Cin=C_tot, f_in=f_tot, rf=None, stride=None, method_deconvolution='oasis') cnm = cnm.fit(images) # DISCARDING A, C, b, f, YrA, sn = cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, cnm.estimates.sn final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_full'] # threshold on time variability fitness_min = params_movie['fitness_delta_min_full'] fitness_delta_min = params_movie['fitness_delta_min_full'] Npeaks = params_movie['Npeaks'] traces = C + YrA idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality( traces, Y, A, C, b, f, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True) ########## A_tot_full = A_tot.tocsc()[:, idx_components] C_tot_full = C_tot[idx_components] comp.comparison['cnmf_full_frame']['timer'] = time.time() - t1 comp.comparison['cnmf_full_frame']['ourdata'] = [ A_tot_full.copy(), C_tot_full.copy() ] #################### ######################## comp.save_with_compare(istruth=False, params=params_movie, Cn=Cn) log_files = glob.glob('*_LOG_*') try: for log_file in log_files: os.remove(log_file) except: logging.warning('Cannot remove log files') ############ assertions ################## pb = False if (comp.information['differences']['params_movie']): logging.error( "you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" ) pb = True if (comp.information['differences']['params_cnm']): logging.warning( "you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" ) # pb = True if (comp.information['diff']['rig']['isdifferent']): logging.error("the rigid shifts are different from the groundtruth ") pb = True if (comp.information['diff']['cnmpatch']['isdifferent']): logging.error( "the cnmf on patch produces different results than the groundtruth " ) pb = True if (comp.information['diff']['cnmfull']['isdifferent']): logging.error( "the cnmf full frame produces different results than the groundtruth " ) pb = True assert (not pb)
dview=dview) else: fname_new = name_new[0] #%% from time import time # fname_new='Yr_d1_501_d2_398_d3_1_order_F_frames_369_.mmap' Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T, ), order='F') #%% # if np.min(images)<0: # raise Exception('Movie too negative, add_to_movie should be larger') #%% if 0: Cn = cm.local_correlations( Y[:, :, ::np.int(old_div(np.shape(images)[0], 3000))]) pl.imshow(Cn) #%% # if not is_patches: # #%% # K = 35 # number of neurons expected per patch # gSig = [7, 7] # expected half size of neurons # merge_thresh = 0.8 # merging threshold, max correlation allowed # p = 2 # order of the autoregressive system # cnm = cnmf.CNMF(n_processes, method_init=init_method, k=K, gSig=gSig, merge_thresh=merge_thresh, # p=p, dview=dview, Ain=None,method_deconvolution='oasis') # cnm = cnm.fit(images) # # %% # else:
filter='*.mmap')[0] if fpath[-3:] == 'nwb': mov = cm.load(cnm_obj.mmap_file, var_name_hdf5='acquisition/TwoPhotonSeries') else: mov = cm.load(cnm_obj.mmap_file) estimates = cnm_obj.estimates params_obj = cnm_obj.params min_mov = np.min(mov) max_mov = np.max(mov) if not hasattr(estimates, 'Cn'): estimates.Cn = cm.local_correlations(mov, swap_dim=False) Cn = estimates.Cn min_mov_denoise = np.min(estimates.A) * estimates.C.min() max_mov_denoise = np.max(estimates.A) * estimates.C.max() background_num = -1 neuron_selected = False nr_index = 0 min_background = np.min(estimates.b, axis=0) * np.min(estimates.f, axis=1) max_background = np.max(estimates.b, axis=0) * np.max(estimates.f, axis=1) if not hasattr(estimates, 'accepted_list'): # if estimates.discarded_components.A.shape[-1] > 0: # estimates.restore_discarded_components() estimates.accepted_list = np.array([], dtype=np.int)
border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory dview.terminate() c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() plt.imshow(Cn,vmax = np.percentile(Cn,95)) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els, nb_patch=2, low_rank_background=False) cnm = cnm.fit(images)
def main(): pass # For compatibility between running under Spyder and the CLI #%% load data fname = os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif') Y = cm.load(fname).astype(np.float32) # # used as a background image Cn = cm.local_correlations(Y.transpose(1, 2, 0)) #%% set up some parameters # frame rate (Hz) fr = 10 # approximate length of transient event in seconds decay_time = 0.5 # expected half size of neurons gSig = [6, 6] # order of AR indicator dynamics p = 1 # minimum SNR for accepting new components min_SNR = 3.5 # correlation threshold for new component inclusion rval_thr = 0.90 # number of background components gnb = 3 # set up some additional supporting parameters needed for the algorithm (these are default values but change according to dataset characteristics) # number of shapes to be updated each time (put this to a finite small value to increase speed) max_comp_update_shape = np.inf # maximum number of expected components used for memory pre-allocation (exaggerate here) expected_comps = 50 # number of timesteps to consider when testing new neuron candidates N_samples = np.ceil(fr * decay_time) # exceptionality threshold thresh_fitness_raw = log_ndtr(-min_SNR) * N_samples # total length of file T1 = Y.shape[0] # set up CNMF initialization parameters # merging threshold, max correlation allowed merge_thresh = 0.8 # number of frames for initialization (presumably from the first file) initbatch = 400 # size of patch patch_size = 32 # amount of overlap between patches stride = 3 # max number of components in each patch K = 4 #%% obtain initial batch file used for initialization # memory map file (not needed) fname_new = Y[:initbatch].save(os.path.join(caiman_datadir(), 'example_movies', 'demo.mmap'), order='C') Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') Cn_init = cm.local_correlations(np.reshape(Yr, dims + (T, ), order='F')) #%% RUN (offline) CNMF algorithm on the initial batch pl.close('all') cnm_init = cnmf.CNMF(2, k=K, gSig=gSig, merge_thresh=merge_thresh, fr=fr, p=p, rf=patch_size // 2, stride=stride, skip_refinement=False, normalize_init=False, options_local_NMF=None, minibatch_shape=100, minibatch_suff_stat=5, update_num_comps=True, rval_thr=rval_thr, thresh_fitness_delta=-50, gnb=gnb, decay_time=decay_time, thresh_fitness_raw=thresh_fitness_raw, batch_update_suff_stat=False, max_comp_update_shape=max_comp_update_shape, expected_comps=expected_comps, dview=None, min_SNR=min_SNR) cnm_init = cnm_init.fit(images) print(('Number of components:' + str(cnm_init.estimates.A.shape[-1]))) pl.figure() crd = plot_contours(cnm_init.estimates.A.tocsc(), Cn_init, thr=0.9) #%% run (online) OnACID algorithm cnm = deepcopy(cnm_init) cnm.params.data['dims'] = (60, 80) cnm._prepare_object(np.asarray(Yr), T1) t = initbatch Y_ = cm.load(fname)[initbatch:].astype(np.float32) for frame_count, frame in enumerate(Y_): cnm.fit_next(t, frame.copy().reshape(-1, order='F')) t += 1 #%% extract the results C, f = cnm.estimates.C_on[gnb:cnm.M], cnm.estimates.C_on[:gnb] A, b = cnm.estimates.Ab[:, gnb:cnm.M], cnm.estimates.Ab[:, :gnb] print(('Number of components:' + str(A.shape[-1]))) #%% pass through the CNN classifier with a low threshold (keeps clearer neuron shapes and excludes processes) use_CNN = True if use_CNN: # threshold for CNN classifier thresh_cnn = 0.1 from caiman.components_evaluation import evaluate_components_CNN predictions, final_crops = evaluate_components_CNN( A, dims, gSig, model_name=os.path.join(caiman_datadir(), 'model', 'cnn_model')) A_exclude, C_exclude = A[:, predictions[:, 1] < thresh_cnn], C[ predictions[:, 1] < thresh_cnn] A, C = A[:, predictions[:, 1] >= thresh_cnn], C[predictions[:, 1] >= thresh_cnn] noisyC = cnm.estimates.noisyC[gnb:cnm.M] YrA = noisyC[predictions[:, 1] >= thresh_cnn] - C else: YrA = cnm.estimates.noisyC[gnb:cnm.M] - C #%% plot results pl.figure() crd = cm.utils.visualization.plot_contours(A, Cn, thr=0.9) view_patches_bar(Yr, A, C, b, f, dims[0], dims[1], YrA, img=Cn)
def save_mc(mc, caiman_fp, is3D): """ DataJoint Imaging Element - CaImAn Integration Run these commands after the CaImAn analysis has completed. This will save the relevant motion correction data into the '*.hdf5' file. Please do not clear variables from memory prior to running these commands. The motion correction (mc) object will be read from memory. 'mc' : CaImAn motion correction object 'caiman_fp' : CaImAn output (*.hdf5) file path 'shifts_rig' : Rigid transformation x and y shifts per frame 'x_shifts_els' : Non rigid transformation x shifts per frame per block 'y_shifts_els' : Non rigid transformation y shifts per frame per block """ # Load motion corrected mmap image mc_image = cm.load(mc.mmap_file, is3D=is3D) # Compute motion corrected summary images average_image = np.mean(mc_image, axis=0) max_image = np.max(mc_image, axis=0) # Compute motion corrected correlation image correlation_image = cm.local_correlations( mc_image.transpose((1, 2, 3, 0) if is3D else (1, 2, 0))) correlation_image[np.isnan(correlation_image)] = 0 # Compute mc.coord_shifts_els grid = [] if is3D: for _, _, _, x, y, z, _ in cm.motion_correction.sliding_window_3d( mc_image[0, :, :, :], mc.overlaps, mc.strides): grid.append([ x, x + mc.overlaps[0] + mc.strides[0], y, y + mc.overlaps[1] + mc.strides[1], z, z + mc.overlaps[2] + mc.strides[2] ]) else: for _, _, x, y, _ in cm.motion_correction.sliding_window( mc_image[0, :, :], mc.overlaps, mc.strides): grid.append([ x, x + mc.overlaps[0] + mc.strides[0], y, y + mc.overlaps[1] + mc.strides[1] ]) # Open hdf5 file and create 'motion_correction' group h5f = h5py.File(caiman_fp, 'r+') h5g = h5f.require_group("motion_correction") # Write motion correction shifts and motion corrected summary images to hdf5 file if mc.pw_rigid: h5g.require_dataset("x_shifts_els", shape=np.shape(mc.x_shifts_els), data=mc.x_shifts_els, dtype=mc.x_shifts_els[0][0].dtype) h5g.require_dataset("y_shifts_els", shape=np.shape(mc.y_shifts_els), data=mc.y_shifts_els, dtype=mc.y_shifts_els[0][0].dtype) if is3D: h5g.require_dataset("z_shifts_els", shape=np.shape(mc.z_shifts_els), data=mc.z_shifts_els, dtype=mc.z_shifts_els[0][0].dtype) h5g.require_dataset("coord_shifts_els", shape=np.shape(grid), data=grid, dtype=type(grid[0][0])) # For CaImAn, reference image is still a 2D array even for the case of 3D # Assume that the same ref image is used for all the planes reference_image = np.tile(mc.total_template_els, (1, 1, correlation_image.shape[-1]))\ if is3D else mc.total_template_els else: h5g.require_dataset("shifts_rig", shape=np.shape(mc.shifts_rig), data=mc.shifts_rig, dtype=mc.shifts_rig[0].dtype) h5g.require_dataset("coord_shifts_rig", shape=np.shape(grid), data=grid, dtype=type(grid[0][0])) reference_image = np.tile(mc.total_template_rig, (1, 1, correlation_image.shape[-1]))\ if is3D else mc.total_template_rig h5g.require_dataset("reference_image", shape=np.shape(reference_image), data=reference_image, dtype=reference_image.dtype) h5g.require_dataset("correlation_image", shape=np.shape(correlation_image), data=correlation_image, dtype=correlation_image.dtype) h5g.require_dataset("average_image", shape=np.shape(average_image), data=average_image, dtype=average_image.dtype) h5g.require_dataset("max_image", shape=np.shape(max_image), data=max_image, dtype=max_image.dtype) # Close hdf5 file h5f.close()