def mov_load_func(x): with warnings.catch_warnings(): if os.path.isdir(x): return load_movie_chain( sorted(glob.glob(os.path.join(x, '*.tif*')))) else: return load_movie_chain([x])
def play_movie(fnames, ds_ratio=0.2, q_max=99.5, fr=30, mag=2): """Play the movie. This will require loading the movie in memory which in general is not needed by the pipeline. Displaying the movie uses the OpenCV library. Press q to close the video panel.""" m_orig = cm.load_movie_chain(fnames) m_orig.resize(1, 1, ds_ratio).play(q_max=q_max, fr=fr, magnification=mag) return
def create_correlation_image(fl_group): import caiman as cm cimg = cm.load_movie_chain(fl_group).local_correlations( eight_neighbours=True, swap_dim=False) # vm = (np.percentile(cimg[~np.isnan(cimg)],90)) # pl.imshow(cimg,vmax = vm) # pl.pause(1) return cimg
def preprocess_neurofinder(folder): import caiman as cm import os fls = glob.glob(os.path.join(folder, 'images/*.tiff')) fls.sort() print(fls[:5]) m = cm.load_movie_chain(fls) m.save(os.path.join(folder, 'movie_total.hdf5'))
def __init__(self, file_name, max_shifts, strides, overlaps, upsample_factor_grid, max_deviation_rigid): self.name_orig = [file_name] self.data_orig = cm.load_movie_chain(self.name_orig) self.name_rig, self.name_pwrig, self.shifts_rig, self.x_shifts_pwrig, self.y_shifts_pwrig, self.template_shape = self._run_motion_correction( file_name, max_shifts, strides, overlaps, upsample_factor_grid, max_deviation_rigid) self.data_rig = cm.load(self.name_rig) self.data_pwrig = cm.load(self.name_pwrig)
def merge_denoised_tiff_files(movie, loaddir, savedir): #%% cpu_num = 2 #cpu_num_spikepursuit = 1 filenames = os.listdir(loaddir) counter = 0 filenames_final = list() residualnames = list() while 'denoised_{}.tif'.format(counter) in filenames: m_new_denoised = cm.load( os.path.join(loaddir, 'denoised_{}.tif'.format(counter))).transpose( 2, 0, 1) i_new_sn = imio.imread( os.path.join(loaddir, 'Sn_image_{}.tif'.format(counter)))[:, :, 0] m_new_trend = cm.load( os.path.join(loaddir, 'trend_{}.tif'.format(counter))).transpose(2, 0, 1) movief = m_new_denoised * i_new_sn + m_new_trend movief.save(os.path.join(loaddir, 'movie{}.tif'.format(counter))) filenames_final.append( os.path.join(loaddir, 'movie{}.tif'.format(counter))) residualnames.append( os.path.join(loaddir, 'PMD_residual_{}.tif'.format(counter))) counter += 1 print(counter) #%% residuals_movie = cm.load_movie_chain(residualnames) residuals_movie.save(os.path.join(savedir, 'PMD_residuals.tif')) #movie_big = cm.load_movie_chain(filenames_final) # %% Memory Mapping c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=cpu_num, single_thread=False) fname_new = cm.save_memmap(filenames_final, base_name=movie['movie_name'], dview=dview, n_chunks=10, order='C') dview.terminate() fname = pathlib.Path(fname_new).name shutil.move(fname_new, os.path.join(savedir, fname)) print('done')
def motion_corr(fnames, dview, opts, disp_movie, is_3d=False): """Perform motion correction""" # Create a motion correction object with the parameters specified. Note # that the file is not loaded in memory mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct(save_movie=True) # Determine maximum shift to be used for trimming against NaNs border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # Compare with original movie if disp_movie and not is_3d: m_els = cm.load(mc.fname_tot_els) m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2).play(fr=60, gain=15, magnification=2, offset=0) # press q to exit return mc, border_to_0
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) # fnames = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.nwb')] fnames = [ os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.nwb') ] # filename to be created or processed # dataset dependent parameters fr = 5 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds starting_time = 0. #%% load the file and save it in the NWB format (if it doesn't exist already) if not os.path.exists(fnames[0]): # fnames_orig = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.h5')] # filename to be processed fnames_orig = [ os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.h5') ] # filename to be processed orig_movie = cm.load(fnames_orig, fr=fr, is3D=True) # orig_movie = cm.load_movie_chain(fnames_orig,fr=fr,is3D=True) # save file in NWB format with various additional info orig_movie.save(fnames[0], sess_desc='test', identifier='demo 3d', exp_desc='demo movie', imaging_plane_description='multi plane', emission_lambda=520.0, indicator='none', location='visual cortex', starting_time=starting_time, experimenter='NAOMi', lab_name='Tank Lab', institution='Princeton U', experiment_description='Experiment Description', session_id='Session 1', var_name_hdf5='TwoPhotonSeries') #%% First setup some parameters for data and motion correction # motion correction parameters dxy = (1., 1., 5.) # spatial resolution in x, y, and z in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (10., 10., 10.) # maximum shift in um patch_motion_um = (50., 50., 30. ) # patch size for non-rigid correction in um # pw_rigid = False # flag to select rigid vs pw_rigid motion correction niter_rig = 1 pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a / b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24, 4) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 is3D = True mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'niter_rig': niter_rig, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy', 'var_name_hdf5': 'acquisition/TwoPhotonSeries', 'is3D': is3D, 'splits_els': 12, 'splits_rig': 12 } opts = params.CNMFParams( params_dict=mc_dict ) #NOTE: default adjustments of parameters are not set yet, manually setting them now # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames, var_name_hdf5=opts.data['var_name_hdf5'], is3D=True) T, h, w, z = m_orig.shape # Time, plane, height, weight m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w)) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing # NOTE: ignore dview right now for debugging purposes # c, dview, n_processes = cm.cluster.setup_cluster( # backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=None, var_name_hdf5=opts.data['var_name_hdf5'], **opts.get_group('motion')) # mc = MotionCorrect(fnames, dview=dview, var_name_hdf5=opts.data['var_name_hdf5'], **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames, var_name_hdf5=opts.data['var_name_hdf5'], is3D=True) T, h, w, z = m_orig.shape # Time, plane, height, weight m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w)) m_els = cm.load(mc.mmap_file, is3D=True) m_els = np.reshape(np.transpose(m_els, (3, 0, 1, 2)), (T * z, h, w)) ds_ratio = 0.2 moviehandle = cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
def main(): pass # For compatibility between running under Spyder and the CLI #%% dr = '/home/nel/Code/Voltage_imaging/exampledata/403106_3min_raw/raw_data/' total = len([f for f in os.listdir(dr) if f.endswith('.tif')]) n_blocks = 1 n_files = np.int(np.floor(total / n_blocks)) for n in range(n_blocks): file_list = [dr + 'cameraTube051_{:05n}.tif'.format(i + 1) for i in np.arange(n_files * n, n_files * (n + 1), 1)] #%% Select file(s) to be processed (download if not present) fnames = [tuple([ff for ff in file_list[:36000]]),tuple([ff for ff in file_list[36000:72000]])] fnames = [tuple([ff for ff in file_list[:72000]])] # fnames = ['/home/nel/Code/Voltage_imaging/exampledata/403106_3min/datasetblock1.hdf5'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% # file_list = ['/home/nel/Code/Voltage_imaging/exampledata/Un-aligned image files/403106_3min/cameraTube051_{:05n}.tif'.format(i+1) for i in range(1000)] # mv = cm.load(file_list) #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (1., 1.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (10., 10.) # maximum shift in um patch_motion_um = (64., 64.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = False # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = False if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True)
# @params num_splits_to_process_els if none all the splits are processed and the movie is saved num_splits_to_process_els = params_movie['num_splits_to_process_els'] # @params upsample_factor_grid upsample factor to avoid smearing when merging patches upsample_factor_grid = params_movie['upsample_factor_grid'] # @params max_deviation_rigid maximum deviation allowed for patch with respect to rigid shift max_deviation_rigid = params_movie['max_deviation_rigid'] # %% download movie if not there if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovieJ.tif']: # TODO: todocument download_demo(fname[0]) fname = [os.path.join('example_movies', fname[0])] # TODO: todocument m_orig = cm.load_movie_chain(fname[:1]) # %% play movie downsample_ratio = params_display['downsample_ratio'] offset_mov = -np.min(m_orig[:100]) m_orig.resize(1, 1, downsample_ratio).play( gain=10, offset=offset_mov, fr=30, magnification=2) # %% RUN ANALYSIS c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %% INITIALIZING t1 = time.time() # movie must be mostly positive for this to work # TODO : document
hdf5File = args.infile[0] outDir = args.outdir[0].replace('\\', '/') online = args.online nSources = args.nsources maskThresh = args.maskthresh inBasename, _ = os.path.splitext(os.path.basename(hdf5File)) outArrayFile = os.path.join(outDir, inBasename) + '_caiman_arrays.npz' outMatlabFile = os.path.join(outDir, inBasename) + '_caiman_arrays.mat' outImageFolder = os.path.join(outDir, inBasename + '_images') if (args.orig_mov) is not None: if type(args.orig_mov) is not list: args.orig_mov = [args.orig_mov] orig_mov = cm.load_movie_chain(args.orig_mov) if not os.path.exists(outDir): os.mkdir(outDir) if not os.path.exists(outImageFolder): os.mkdir(outImageFolder) print('Input hdf5 file: {}'.format(hdf5File), flush=True) print('Output directory: {}'.format(outDir), flush=True) # load in data print('Loading data', flush=True) if args.online: cnm = cnmf.online_cnmf.load_OnlineCNMF(hdf5File)
expt_names = [ './lateral_videos/PS3_Vid1_conv.tif', './lateral_videos/PS3_Vid14_conv.tif' ] expt_fold = [os.path.split(nm)[0].split('/')[1] for nm in gt_names] print(expt_names) print(expt_fold) for e, e1 in zip(gt_names, expt_names): print(e) pts = scipy.io.loadmat(e)['points'][0][0] lat_sess = scipy.signal.savgol_filter( np.sqrt( np.diff(np.array(pts[0]).T, axis=0)**2 + np.diff(np.array(pts[8]).T, axis=0)**2), 3, 1)[2:] mat_files = [e1] m = cm.load_movie_chain(mat_files[:1], fr=100)[2:] pl.figure() pl.imshow(m[10], cmap='gray') pl.plot(pts[0][10] / 3, pts[8][10] / 3, 'r*') if mask_all: mask = np.ones(m.shape[1:]) np.save(e[:-4] + '_mask_all_lat.npy', mask) else: mask = behavior.select_roi(np.median(m[::100], 0), 1)[0] np.save(e[:-4] + '_mask_lat_hl.npy', mask) #%% r_values = [] only_magnitude = False n_components = 6
if save_movie: out.release() out = None cv2.destroyAllWindows() #%% save results (optional) save_results = False if save_results: np.savez('results_analysis_online_MOT_CORR.npz', Cn=Cn, Ab=cnm2.Ab, Cf=cnm2.C_on, b=cnm2.b, f=cnm2.f, dims=cnm2.dims, tottime=tottime, noisyC=cnm2.noisyC, shifts=shifts) #%% create correlation image or raw data with applied shifts Y_off = cm.load_movie_chain(fls)[200:].apply_shifts(shifts,interpolation='cubic') Cn = Y_off.local_correlations(swap_dim=False) pl.figure(); crd = cm.utils.visualization.plot_contours(cnm2.Ab[:,cnm2.gnb:], Cn, thr=0.95, display_numbers = False) #%% extract results from the objects and do some plotting A, b = cnm2.Ab[:, cnm2.gnb:], cnm2.Ab[:, :cnm2.gnb].toarray() C, f = cnm2.C_on[cnm2.gnb:cnm2.M, t-t//epochs:t], cnm2.C_on[:cnm2.gnb, t-t//epochs:t] noisyC = cnm2.noisyC[:,t-t//epochs:t] b_trace = [osi.b for osi in cnm2.OASISinstances] if hasattr(cnm2, 'OASISinstances') else [0]*C.shape[0] pl.figure() crd = cm.utils.visualization.plot_contours(A, Cn, thr=0.9) view_patches_bar(Yr, scipy.sparse.coo_matrix(A.tocsc()[:, :]), C[:, :], b, f, dims[0], dims[1], YrA=noisyC[cnm2.gnb:cnm2.M] - C, img=Cn)
if save_results: np.savez('results_analysis_online_MOT_CORR.npz', Cn=Cn, Ab=cnm2.Ab, Cf=cnm2.C_on, b=cnm2.b, f=cnm2.f, dims=cnm2.dims, tottime=tottime, noisyC=cnm2.noisyC, shifts=shifts) #%% create correlation image or raw data with applied shifts Y_off = cm.load_movie_chain(fls)[200:].apply_shifts(shifts, interpolation='cubic') Cn = Y_off.local_correlations(swap_dim=False) pl.figure() crd = cm.utils.visualization.plot_contours(cnm2.Ab[:, cnm2.gnb:], Cn, thr=0.95, display_numbers=False) #%% extract results from the objects and do some plotting A, b = cnm2.Ab[:, cnm2.gnb:], cnm2.Ab[:, :cnm2.gnb].toarray() C, f = cnm2.C_on[cnm2.gnb:cnm2.M, t - t // epochs:t], cnm2.C_on[:cnm2.gnb, t - t // epochs:t] noisyC = cnm2.noisyC[:, t - t // epochs:t] b_trace = [osi.b for osi in cnm2.OASISinstances] if hasattr( cnm2, 'OASISinstances') else [0] * C.shape[0]
if mot_correct: # Motion correction setup #------------------------------------------------------------------------------- mc = MotionCorrect(tf, dview=None, **opts.get_group('motion')) # Run piecewise-rugu nituib cirrectuib ysubg NoRMCorre #------------------------------------------------------------------------------- mc.motion_correct(save_movie=True) m_els = cm.load(mc.fname_tot_els) border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # maximum shift to be used for trimming against NaNs # Compare wth original movie - This isn't currently working particularly well #------------------------------------------------------------------------------- if display_movie: m_orig = cm.load_movie_chain(tf) ds_ratio = 0.2 cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2).play(fr=60, gain=15, magnification=2, offset=0) # press q to exit # Memory Mapping #------------------------------------------------------------------------------- mapfile = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C') Yr, dims, T = cm.load_memmap(mapfile) images = np.reshape(Yr.T, [T] + list(dims), order='F') # compute some summary images (correlation and peak to noise)
# if ds_factor != 1: # m=m.resize(ds_factor,ds_factor,1) # if len(Cn) == 0: # Cn = np.zeros(m.shape[1:]) # # Cn = np.maximum(Cn,m.local_correlations(swap_dim=False)) # pl.imshow(Cn,cmap='gray') # pl.pause(.1) #%% # Cn = Y.local_correlations(eight_neighbours=True, swap_dim=False) # np.save('Cn_90k.npy',Cn) #%% if ds > 1: Y = cm.load_movie_chain(fls[:init_files])[ :initbatch].resize(1. / ds, 1. / ds) else: Y = cm.load_movie_chain(fls[:init_files])[:initbatch] if mot_corr: mc = Y.motion_correct(max_shift, max_shift) Y = mc[0].astype(np.float32) borders = np.max(mc[1]) else: Y = Y.astype(np.float32) if ds_factor != 1: Y = Y.resize(ds_factor, ds_factor, 1) gSig = 1 + (np.multiply(gSig, ds_factor)).astype(np.int) img_min = Y.min()
from caiman.source_extraction.cnmf import params as params from caiman.utils.utils import download_demo from caiman.summary_images import local_correlations_movie_offline #%% start by loading basic confiugurations logging.basicConfig(format= "%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s", # filename="/tmp/caiman.log", level=logging.WARNING) #%% os.getcwd() to figure out where you are and os.chdir() to your directory with the movies fnames = ['test.tif'] # filename to be processed #%%Check your movie first display_movie = True # False if you dont want to see it. Press q to exit if display_movie: m_orig = cm.load_movie_chain(fnames) #this parameters work for a 256X256 for the 2p ds_ratio = 0.2 m_orig.resize(1, 1, ds_ratio).play( q_max=85, fr=3, magnification=2) # pick your fr and change your q_max appropiatetly #%% Start building the parameters (I will build all parameters for future uses like CNMF not only mc) #%% build parameters # dataset dependent parameters fr = 3 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # parameters for source extraction and deconvolution p = 0 # order of the autoregressive system (p=0 deconvolution off) gnb = 2 # number of global background components merge_thr = 0.95 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
import os import caiman as cm import argparse parser = argparse.ArgumentParser( description='save multiple video files as a single tif') parser.add_argument('infile', type=str, nargs=1, help='.txt file containing chunk file locations') parser.add_argument('outfile', type=str, nargs=1, help='output file name (.tif file name should be used)') args = vars(parser.parse_args()) infile = args['infile'][0] outfile = args['outfile'][0] if '.txt' in infile: with open(infile, 'r') as f: flist = [] for line in f: flist = flist + [line.replace('\n', '')] m = cm.load_movie_chain(flist) m.save(outfile)
# @params fname name of the movie fname = params_movie['fname'] niter_rig = params_movie['niter_rig'] # @params max_shifts maximum allow rigid shift max_shifts = params_movie['max_shifts'] # @params splits_rig for parallelization split the movies in num_splits chuncks across time splits_rig = params_movie['splits_rig'] # @params num_splits_to_process_ri if none all the splits are processed and the movie is saved num_splits_to_process_rig = params_movie['num_splits_to_process_rig'] # %% download movie if not there channel = 0 m_orig = cm.load_movie_chain(fname[:1], channel=(slice(None), channel, slice(None), slice(None))) m_orig.save(fname[0][:-4] + '_channel_' + str(channel) + '.tif') fname[0] = fname[0][:-4] + '_channel_' + str(channel) + '.tif' # %% play movie downsample_ratio = params_display['downsample_ratio'] offset_mov = -np.min(m_orig[:100]) m_orig.resize(1, 1, downsample_ratio).play(gain=10, offset=offset_mov, fr=30, magnification=5) # %% RUN ANALYSIS c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False)
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set this to true to show movies and plots fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. To # close the video press q m_orig = cm.load_movie_chain(fname[:1]) downsample_ratio = 0.2 offset_mov = -np.min(m_orig[:100]) moviehandle = m_orig.resize(1, 1, downsample_ratio) if display_images: moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # maximum shift to be used for trimming against NaNs #%% compare with original movie moviehandle = cm.concatenate([ m_orig.resize(1, 1, downsample_ratio) + offset_mov, m_els.resize(1, 1, downsample_ratio) ], axis=2) display_images = False if display_images: moviehandle.play(fr=60, q_max=99.5, magnification=2, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) #%% PLOT COMPONENTS if display_images: plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) if display_images: view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=Cn) view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components_bad], cnm.C[idx_components_bad], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components_bad], img=Cn) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape( dims + (-1, ), order='F').transpose([2, 0, 1]) #%% play along side original data moviehandle = cm.concatenate([ m_els.resize(1, 1, downsample_ratio), denoised.resize(1, 1, downsample_ratio) ], axis=2) if display_images: moviehandle.play(fr=60, gain=15, magnification=2, offset=0) # press q to exit
def piecewise_rigid_registration(fnames, max_shifts=(6, 6), strides=(48, 48), overlaps=(24, 24), num_frames_split=100, max_deviation_rigid=3, pw_rigid=False, shifts_opencv=True, border_nan='copy', highPassFilter=False, sigma=7): '''Apply CaImAn piecewise rigid registration on raw imaging data :param fnames: full path filename for motion correction, possible extensions are tif, avi, npy :param max_shifts: maximum allowed rigid shift in pixels (view the movie to get a sense of motion) :param strides: create a new patch every x pixels for pw-rigid correction :param overlaps: overlap between pathes (size of patch strides+overlaps) :param num_frames_split: length in frames of each chunk of the movie (to be processed in parallel) :param max_deviation_rigid: maximum deviation allowed for patch with respect to rigid shifts :param pw_rigid: flag for performing rigid or piecewise rigid motion correction :param shifts_opencv: flag for correcting motion using bicubic interpolation (otherwise FFT interpolation is used) :param border_nan: replicate values along the boundary (if True, fill in with NaN) :param highPassFilter: whether applying high pass filter :param sigma: lowpass gaussian filter sigma value :return: motion corrected image in numpy ''' fnames = [fnames] m_orig = cm.load_movie_chain(fnames) if 'dview' in locals(): cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # create a motion correction object mc = MotionCorrect(fnames, dview=dview, max_shifts=max_shifts, strides=strides, overlaps=overlaps, max_deviation_rigid=max_deviation_rigid, shifts_opencv=shifts_opencv, nonneg_movie=True, border_nan=border_nan) # correct for rigid motion correction and save the file (in memory mapped form) mc.motion_correct(save_movie=True) # load motion corrected movie and apply piece-wise rigid registration m_rig = cm.load(mc.mmap_file) mc.pw_rigid = True # turn the flag to True for pw-rigid motion correction mc.template = mc.mmap_file # use the template obtained before to save in computation (optional) mc.motion_correct(save_movie=True, template=mc.total_template_rig) cm.stop_server(dview=dview) # stop the server m_els = cm.load(mc.fname_tot_els) Y = np.array(m_els.tolist()) if highPassFilter: Y = high_pass_filtering(Y, sigma) return Y
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters for data and motion correction # dataset dependent parameters fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid motion correction in um # motion correction parameters pwrigid_motion_correct = True # flag to select rigid vs pw_rigid motion correction max_shifts = tuple([int(a/b) for a, b in zip(max_shift_um, dxy)]) # maximum allow rigid shift in pixels # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = False if display_images: m_orig = cm.load_movie_chain(fname) downsample_ratio = 0.2 moviehandle = m_orig.resize(1, 1, downsample_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname, min_mov, dview=dview, max_shifts=max_shifts, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, border_nan='copy', upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre if pwrigid_motion_correct: mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. else: mc.motion_correct_rigid(save_movie=True) m_els = cm.load(mc.fname_tot_rig) bord_px_els = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) fnames = mc.fname_tot_rig # name of the rigidly corrected file. # maximum shift to be used for trimming against NaNs #%% compare with original movie if display_images: downsample_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, downsample_ratio) - min_mov, m_els.resize(1, 1, downsample_ratio)], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' border_to_0 = bord_px_els # exclude borders due to motion correction # border_to_0 = 0 if mc.border_nan is 'copy' else bord_px_els # you can include boundaries if you used the 'copy' option in the motion # correction, although be careful abou the components near the boundaries fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' # parameters for component evaluation opts = params.CNMFParams(dims=dims, fr=fr, decay_time=decay_time, method_init=method_init, gSig=gSig, merge_thresh=merge_thresh, p=p, gnb=gnb, k=K, rf=rf, stride=stride_cnmf, rolling_sum=True) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.set('temporal', {'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier cnm.params.set('quality', {'fr': fr, 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr}) cnm.estimates.evaluate_components(images, cnm.params, dview=dview) #%% PLOT COMPONENTS cnm.estimates.plot_contours(img=Cn, idx=cnm.estimates.idx_components) #%% VIEW TRACES (accepted and rejected) if display_images: cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components) cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components_bad) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.dview = None cnm2 = deepcopy(cnm) cnm2.dview = dview cnm2.params.set('patch', {'rf': None}) cnm2.params.set('temporal', {'p': p}) cnm2 = cnm2.fit(images) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(Yr, img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=True) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
import caiman as cm import os import numpy as np os.chdir('C:\\Users\\grossar\\Data\\Caiman\\CaImAn\\') #source activate CaImAn ### Loading movies single_movie = cm.load('example_movie\\demoMovie.tif') print(single_movie.shape) file_names = ['example_movies/demoMovie.tif', 'example_movies/demoMovie.tif' ] # for the sake of the example we repeat the same movie movies_chained = cm.load_movie_chain(file_names) print(movies_chained.shape) ### Generating a random movie movie_random = cm.movie(np.random.random([1000, 100, 100])) ### Saving Movies movie_random.save('movie_random-10-29.tif') np.random.random([2, 3, 3]) ### Visualizing Movies movies_chained.play(magnification=2, fr=30, q_min=0.1, q_max=99.75)
'./gc-AG052014-02/062314_-213 -218 67_WHISK_COND_A__2/points_trial1.mat', './AG052014-01/063014_98 -639 144_COND_A__1/points_trial9.mat', './AG052014-01/070214_79 -645 131_COND_A_/points_trial99.mat'] expt_fold = [os.path.split(nm)[0].split('/')[1] for nm in gt_names] expt_names = [os.path.split(nm)[0].split('/')[2] for nm in gt_names] print(expt_names) print(expt_fold) #%% for e in gt_names: print(e) pts = scipy.io.loadmat(e)['points'][0][0] whisk_sess = scipy.signal.savgol_filter( np.abs(np.diff(np.array(pts[8]).T, axis=0)), 3, 1)[2:] num_tr = str(re.findall('\d+', e[-8:-4])[0]) mat_files = [os.path.join(os.path.split(e)[0], 'trial' + num_tr + '.mat')] m = cm.load_movie_chain(mat_files[:1], fr=100)[2:] pl.figure() pl.imshow(m[0], cmap='gray') pl.plot(pts[0][10] / 3, pts[8][10] / 3, 'r*') mask = behavior.select_roi(np.median(m[::100], 0), 1)[0] if mask_all: np.save(e[:-4] + '_mask_all.npy', mask) else: np.save(e[:-4] + '_mask_whisk.npy', mask)
# if ds_factor != 1: # m=m.resize(ds_factor,ds_factor,1) # if len(Cn) == 0: # Cn = np.zeros(m.shape[1:]) # # Cn = np.maximum(Cn,m.local_correlations(swap_dim=False)) # pl.imshow(Cn,cmap='gray') # pl.pause(.1) #%% # Cn = Y.local_correlations(eight_neighbours=True, swap_dim=False) # np.save('Cn_90k.npy',Cn) #%% if ds > 1: Y = cm.load_movie_chain(fls[:init_files])[:initbatch].resize( 1. / ds, 1. / ds) else: Y = cm.load_movie_chain(fls[:init_files])[:initbatch] if mot_corr: mc = Y.motion_correct(max_shift, max_shift) Y = mc[0].astype(np.float32) borders = np.max(mc[1]) else: Y = Y.astype(np.float32) if ds_factor != 1: Y = Y.resize(ds_factor, ds_factor, 1) gSig = 1 + (np.multiply(gSig, ds_factor)).astype(np.int) img_min = Y.min() Y -= img_min
# process input file if '.txt' in infile[0]: with open(infile[0], 'r') as f: flist = [] for line in f: flist = flist + [line.replace('\n','')] fname = flist elif os.path.isdir(infile[0]): tmpMovPath = os.path.join(infile[0], 'tmp_mov.tif') if not os.path.isfile(tmpMovPath): tfiles = glob.glob( os.path.join(infile[0], '*.tiff')) + glob.glob( os.path.join(infile[0], '*.tif')) tfiles = sorted(list(set(tfiles))) with warnings.catch_warnings(): tmpMov = cm.load_movie_chain(tfiles) tmpMov.save(tmpMovPath) del tmpMov fname = [tmpMovPath] else: fname = infile print('Using file(s): {}'.format(fname)) # get file extension (of first file) _, fext = os.path.splitext(fname[0]) ## process template into ndarray if mc_temp is not None: mc_temp = tif.imread(mc_temp) if len(mc_temp.shape)>2:
def main(): pass # For compatibility between running under Spyder and the CLI #%% """ General parameters """ play_movie = 1 plot_extras = 1 plot_extras_cell = 1 compute_mc_metrics = 1 #%% Select file(s) to be processed (download if not present) """ Load file """ #fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed #if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: # fnames = [download_demo(fnames[0])] #fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif'] #f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\' f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\' f_name = 'Ch1' f_ext = 'tif' fnames = [f_dir + f_name + '.' + f_ext] #fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5'] #%% First setup some parameters for data and motion correction """ Parameters """ # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 1 #0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels #max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] max_shifts = [6, 6] # start a new patch for pw-rigid motion correction every x pixels #strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) strides = [48, 48] # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q if play_movie: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # type "mc."and press TAB to see all interesting associated variables and self. outputs # interesting outputs # saved file is mc.fname_tot_els / mc.fname_tot_rig # mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch # mc.coord_shifts_els: coordinates associated to patches with shifts # mc.total_template_els: updated template for pw # mc.total_template_rig: updated template for rigid # mc.templates_rig: templates for each iteration in rig #%% # compute metrics for the results (TAKES TIME!!) if compute_mc_metrics: # not finished bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) final_size = np.subtract( mc.total_template_els.shape, 2 * bord_px_els) # remove pixels in the boundaries winsize = 100 swap_dim = False resize_fact_flow = .2 # downsample for computing ROF tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction( fnames[0], final_size[0], final_size[1], swap_dim, winsize=winsize, play_flow=False, resize_fact_flow=resize_fact_flow) plt.figure() plt.plot(correlations_orig) # %% compare with original movie if play_movie: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit del m_orig del m_els if plot_extras: # plot total template plt.figure() plt.imshow(mc.total_template_els) plt.title('Template after iteration') # plot x and y corrections plt.figure() plt.plot(mc.shifts_rig) plt.title('Rigid motion correction xy movement') plt.legend(['x shift', 'y shift']) plt.xlabel('frames') # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 2 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 2 # number of components per patch gSig = [15, 15] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 1 # spatial subsampling during initialization tsub = 1 # temporal subsampling during intialization # parameters for component evaluation opts_dict = { 'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub } opts.change_params(params_dict=opts_dict) # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) if plot_extras_cell: num_cell_plot = 51 plt.figure() plt.plot(cnm.estimates.C[num_cell_plot, :]) plt.title('Temporal component') plt.legend(['Cell ' + str(num_cell_plot)]) # plot component sptial profile A # first convert back to dense components plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape( list(dims)) plt.figure() plt.imshow(plot_spat_A) plt.title('Spatial component cell ' + str(num_cell_plot)) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') if plot_extras: plt.figure() plt.imshow(Cn) plt.title('Local correlations') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.90 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Component selection: min_SNR=' + str(min_SNR) + '; rval_thr=' + str(rval_thr) + '; cnn prob range=[' + str(cnn_lowest) + ' ' + str(cnn_thr) + ']') # %% VIEW TRACES (accepted and rejected) if plot_extras: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Accepted') cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) plt.suptitle('Rejected') #plt.figure(); #plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:]) # # # # #plt.figure(); #plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]); #plt.plot(); #plt.show(); # # #plt.figure(); #plt.plot(cnm2.estimates.detrend_df_f[1,:]) # these store the good and bad components, and next step sorts them # cnm2.estimates.idx_components # cnm2.estimates.idx_components_bad #%% update object with selected components #cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) plt.suptitle("Final results") #%% Save the mc data as in cmn struct as well ## #mc_out = dict( # pw_rigid = mc.pw_rigid, # fname = mc.fname, # mmap_file = mc.mmap_file, # total_template_els = mc.total_template_els, # total_template_rig = mc.total_template_rig, # border_nan = mc.border_nan, # border_to_0 = mc.border_to_0, # x_shifts_els = mc.x_shifts_els, # y_shifts_els = mc.y_shifts_els, # Cn = Cn # ) # # #deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out) #%% reconstruct denoised movie (press q to exit) if play_movie: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) save_results = True if save_results: cnm2.save(fnames[0][:-4] + '_results.hdf5')
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) root = '/Users/hheiser/Desktop/testing data/chronic_M2N3/0d_baseline/channel1' fnames = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\1\file_00003.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\2\file_00004.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\3\file_00005.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\4\file_00006.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\5\file_00007.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\6\file_00008.tif'] fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) #%% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=30, magnification=1, do_loop=False) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory #%% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) #%% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames[:3]) m_els = cm.load(mmap_file[:3]) ds_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie, m_els.resize(1, 1, ds_ratio)], axis=2) moviehandle.play(fr=15, q_max=99.5, magnification=2) # press q to exit #%% MEMORY MAPPING border_to_0 = 0 if mc.border_nan == 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries mmap_file = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\1\file_00001_els__d1_512_d2_512_d3_1_order_F_frames_1147_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\2\file_00002_els__d1_512_d2_512_d3_1_order_F_frames_2520_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\3\file_00003_els__d1_512_d2_512_d3_1_order_F_frames_3814_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\4\file_00004_els__d1_512_d2_512_d3_1_order_F_frames_5154_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\5\file_00005_els__d1_512_d2_512_d3_1_order_F_frames_2677_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\6\file_00006_els__d1_512_d2_512_d3_1_order_F_frames_3685_.mmap'] fname_new = r'E:\PhD\Data\DG\M14_20191014\N1\memmap__d1_512_d2_512_d3_1_order_C_frames_34939_.mmap' # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = {'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub} opts.change_params(params_dict=opts_dict) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) #opts.change_params({'p': 1,'rf':None, 'only_init':False}) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) #%% RUN CNMF SEEDED WITH MANUAL MASK # load mask mask = np.asarray(imageio.imread('/Users/hheiser/Desktop/testing data/file_00020_no_motion/avg_mask_fixed.png'), dtype=bool) # get component ROIs from the mask and plot them Ain, labels, mR = cm.base.rois.extract_binary_masks(mask) # plot original mask and extracted labels to check mask fig, ax = plt.subplots(1,2) ax[0].imshow(-mR,cmap='binary') ax[0].set_title('Original mask') ax[1].imshow(labels) ax[1].set_title('Extracted labelled ROIs') """" plt.figure() crd = cm.utils.visualization.plot_contours( Ain.astype('float32'), mR, thr=0.99, display_numbers=True) # todo check if this is important for the pipeline plt.title('Contour plots of detected ROIs in the structural channel') """ opts.change_params({'rf': None, 'only_init': False}) # run CNMF seeded with this mask cnm_corr = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain) cnm_corr_seed = cnm_corr_seed.fit(images) #cnm_seed = cnm_seed.fit_file(motion_correct=False) #%% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) #%% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.evaluate_components(images, params=cnm2.params, dview=dview) #%% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) #%% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components #### -> will delete rejected components! cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=1, bpx=border_to_0, include_bck=True) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% save results dirname = fnames[0][:-4] + "_results.hdf5" cnm2.estimates.Cn = Cn cnm2.save(dirname) #load results cnm2 = cnmf.load_CNMF(dirname) mov_name = fnames[0][:-4] + "_movie_restored_2_gain.tif" helper.save_movie(cnm2.estimates,images,mov_name,frame_range=range(200),include_bck=True)
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie, m_els.resize(1, 1, ds_ratio)], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = {'fnames': fnames, 'p': p, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub} opts.change_params(params_dict=opts_dict); # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0). If you want to have # deconvolution within each patch change params.patch['p_patch'] to a # nonzero value #opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cns = local_correlations_movie_offline(mc.mmap_file[0], remove_baseline=True, window=1000, stride=1000, winSize_baseline=100, quantil_min_baseline=10, dview=dview) Cn = Cns.max(axis=0) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% save results cnm.estimates.Cn = Cn cnm.save(fname_new[:-5]+'_init.hdf5') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% cnm2.estimates.Cn = Cn cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
# @params fname name of the movie fname = params_movie['fname'] niter_rig = params_movie['niter_rig'] # @params max_shifts maximum allow rigid shift max_shifts = params_movie['max_shifts'] # @params splits_rig for parallelization split the movies in num_splits chuncks across time splits_rig = params_movie['splits_rig'] # @params num_splits_to_process_ri if none all the splits are processed and the movie is saved num_splits_to_process_rig = params_movie['num_splits_to_process_rig'] # %% download movie if not there channel = 0 m_orig = cm.load_movie_chain(fname[:1], channel=( slice(None), channel, slice(None), slice(None))) m_orig.save(fname[0][:-4] + '_channel_' + str(channel) + '.tif') fname[0] = fname[0][:-4] + '_channel_' + str(channel) + '.tif' # %% play movie downsample_ratio = params_display['downsample_ratio'] offset_mov = -np.min(m_orig[:100]) m_orig.resize(1, 1, downsample_ratio).play( gain=10, offset=offset_mov, fr=30, magnification=5) # %% RUN ANALYSIS c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %% INITIALIZING t1 = time.time() # movie must be mostly positive for this to work
import sys,os import pickle import pandas as pd #%% motion correction with concatenated videos videoconcat = sys.argv[1] newpath = os.path.dirname(videoconcat) ms_ts_name = os.path.join(newpath,"ms_ts.pkl") if os.path.exists(ms_ts_name): with open(ms_ts_name, "rb") as f: ms_ts= pickle.load(f) else: print("there is no mt_ts.pkl existed") fnames=[videoconcat] m_orig = cm.load_movie_chain(fnames) # start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened) if 'dview' in locals(): cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # dataset dependent parameters fr = 30 # movie frame rate decay_time = 0.4 # length of a typical transient in seconds motion_correct = True # flag for motion correction # motion correction parameters pw_rigid = False # flag for pw-rigid motion correction gSig_filt = (8,8) # size of filter, in general gSig (see below),
def get_mc_movie(self): path = self.get_mc_movie_path() return cm.load_movie_chain([str(path)])