def init_mc_params(self, **kwargs): """ Define parameters. """ #Build the dict. These are defaults. params_dict = { 'fnames': self.fnames, # file names 'fr': 20, # frame rate 'decay_time': .4, # transient decay time 'pw_rigid': False, # flag for performing piecewise-rigid motion correction (otherwise just rigid) 'max_shifts': (5, 5), # maximum allowed rigid shift 'gSig_filt': (3, 3), # size of high pass spatial filtering, used in 1p data 'strides': ( 48, 48 ), # start a new patch for pw-rigid motion correction every x pixels 'overlaps': (24, 24), # overlap between pathes (size of patch strides+overlaps) 'max_deviation_rigid': 3, # maximum deviation allowed for patch with respect to rigid shifts 'border_nan': 'copy', # replicate values along the boundaries } # If any parameters were modified from default, this line reflects those changes. for key, value in kwargs.items(): params_dict[key] = value opts = params.CNMFParams(params_dict=params_dict) return opts
def set_opts(fnames, fr, decay_time, opts_dict): """Parameters not defined in the dictionary will assume their default values. The resulting params object is a collection of subdictionaries pertaining to the dataset to be analyzed (params.data), motion correction (params.motion), data pre-processing (params.preprocess), initialization (params.init), patch processing (params.patch), spatial and temporal component (params.spatial), (params.temporal), quality evaluation (params.quality) and online processing (params.online)""" opts_dict = { **opts_dict, 'fnames': fnames, 'fr': fr, 'decay_time': decay_time } opts = params.CNMFParams(params_dict=opts_dict) return opts
def __init__(self, mc_opts, tiff, channels, planes, x_start, x_end): self.tiff = tiff self.channels = channels self.planes = planes self.x_start = x_start self.x_end = x_end self.xslice = slice(x_start, x_end) self.mc_opts = mc_opts self.opts = params.CNMFParams(params_dict=mc_opts) self.c, self.dview, self.n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) self.file_list = [] self.images = [] self.motion_corrected_images = [] self.masks = []
def motion_correct_file(folder, dview): import caiman as cm from caiman.motion_correction import MotionCorrect from caiman.source_extraction.cnmf import params as params print(folder) mc_dict = { 'fnames': [os.path.join(folder, 'movie_total.hdf5')], 'fr': 7, 'decay_time': 1.5, 'dxy': [1.15, 1.15], 'pw_rigid': False, 'max_shifts': [5,5], 'strides': None, 'overlaps': None, 'max_deviation_rigid': None, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) mc = MotionCorrect(mc_dict['fnames'], dview=dview, **opts.get_group('motion')) mc.motion_correct(save_movie=True)
def __init__(self, caiman_params, channels, planes, x_start, x_end, folder, batch_size=15): self.channels = channels self.planes = planes self.x_start = x_start self.x_end = x_end self.folder = folder self.caiman_params = caiman_params # derived params self.folder_tiffs = folder + '*.tif*' self.save_folder = folder + 'out/' print('Setting up caiman...') self.opts = params.CNMFParams(params_dict=self.caiman_params) self.batch_size = batch_size # can be overridden by expt runner self.fnumber = 0 self._splits = None self._json = None self.times = None self.cond = None self.vis_cond = None # other init things to do # start server self._start_cluster() # cleanup cleanup(self.folder, 'mmap') cleanup(self.save_folder, 'hdf5') cleanup(self.save_folder, 'json') cleanup(os.getcwd(), 'npz')
def __init__(self, tiff, channels, planes, x_start, x_end, mc_opts, use_green_ch=False,): self.tiff = tiff self.channels = channels self.planes = planes self.x_start = x_start self.x_end = x_end self.xslice = slice(x_start, x_end) self.mc_opts = mc_opts self.use_green_ch = use_green_ch if self.use_green_ch == True: self.channel2use = 0 else: self.channel2use = 1 if self.mc_opts: self.opts = params.CNMFParams(params_dict=mc_opts) self.file_list = [] self.images = [] self.motion_corrected_images = [] self.masks = [] self.coords = None self.corr_images = []
def cnmf_neurofinder(params_dict): import numpy as np from caiman.source_extraction.cnmf import cnmf from caiman.source_extraction.cnmf import params fname_new = params_dict['fnames'][0] print(fname_new) Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') opts = params.CNMFParams(params_dict=params_dict) dview = params_dict['dview'] print('Starting CNMF') opts.set('temporal', {'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) cnm.params.change_params({'update_background_components': True, 'skip_refinement': False, 'n_pixels_per_process': 4000, 'dview': dview}) opts.set('temporal', {'p': params_dict['p']}) cnm2 = cnm.refit(images, dview=dview) cnm2.save(fname_new[:-5] + '_cnmf.hdf5') return cnm2
def __init__(self, caiman_params, channels, planes, x_start, x_end, folder): self.channels = channels self.planes = planes self.x_start = x_start self.x_end = x_end self.folder = folder self.caiman_params = caiman_params self.trial_lengths = [] # derived params self.folder_tiffs = folder + '*.tif*' try: if not os.path.exists(folder + 'out/'): os.mkdir(folder + 'out/') except OSError: print("can't make the save path for some reason :( ") self.save_folder = folder + 'out/' print('Setting up caiman...') self.opts = params.CNMFParams(params_dict=self.caiman_params) self.batch_size = 15 # can be overridden by expt runner self.fnumber = 0 self.bad_tiff_size = 10 self._splits = None self._json = None # other init things to do # start server self._start_cluster() # cleanup cleanup_mmaps(self.folder) cleanup_hdf5(self.save_folder) cleanup_json(self.save_folder) self._everything_is_OK = True
#print("param_dict param does not exist, with error: {}".format(e)) raise OSError("no param dict!") try: cnn = configparams["cnn"] download(bucketname, cnn, locname + "/") path_cnn = os.path.join(locname, os.path.basename(path_paramdict)) paramdict["online"]["path_to_model"] = path_cnn print(paramdict) except Exception as e: print("cnn param does not exist!, error: {}".format(e)) paramdict["online"][ "path_to_model"] = "/home/ubuntu/caiman_data/model/cnn_model_online.h5" ## If param_mode is simple, write the given parameters into a new caiman parameter dictionary. elif param_mode == "simple": paramset = params.CNMFParams() try: other_params = configparams["params"] for key in other_params: paramset.change_params({key: other_params[key]}) paramdict = paramset.to_dict() paramdict["online"][ "path_to_model"] = "/home/ubuntu/caiman_data/model/cnn_model_online.h5" except Exception as e: print("params not given") raise OSError("params not given correctly.") else: print("param mode {} not recognized. exiting.".format(param_mode)) raise OSError("param dict not given correctly.")
def run( file_path, n_cpus, motion_correct: bool = True, mc_settings: dict = {}, job_name: str = "job", output_directory: str = "", ): mkl = os.environ.get("MKL_NUM_THREADS") blas = os.environ.get("OPENBLAS_NUM_THREADS") vec = os.environ.get("VECLIB_MAXIMUM_THREADS") print(f"MKL: {mkl}") print(f"blas: {blas}") print(f"vec: {vec}") # we import the pipeline upon running so they aren't required for all installs import caiman as cm from caiman.motion_correction import MotionCorrect from caiman.source_extraction.cnmf import params as params # print the directory caiman is imported from caiman_path = os.path.abspath(cm.__file__) print(f"caiman path: {caiman_path}") sys.stdout.flush() # setup the logger logger_file = os.path.join(output_directory, "caiman.log") logging.basicConfig( format=LOGGER_FORMAT, filename=logger_file, filemode="w", level=logging.DEBUG, ) # if indices to perform mcorr are set, format them if "indices" in mc_settings: indices = mc_settings["indices"] indices_formatted = () for axis_slice in indices: start = axis_slice[0] stop = axis_slice[1] if len(axis_slice) == 3: step = axis_slice[2] else: step = 1 indices_formatted += (slice(start, stop, step), ) mc_settings["indices"] = indices_formatted # load and update the pipeline settings mc_parameters = DEFAULT_MCORR_SETTINGS for k, v in mc_settings.items(): mc_parameters[k] = v opts = params.CNMFParams(params_dict=mc_parameters) # get the filenames if os.path.isfile(file_path): print(file_path) fnames = [file_path] else: file_pattern = os.path.join(file_path, "*.tif*") fnames = sorted(glob.glob(file_pattern)) print(fnames) opts.set("data", {"fnames": fnames}) if n_cpus > 1: print("starting server") # start the server n_proc = np.max([(n_cpus - 1), 1]) c, dview, n_processes = cm.cluster.setup_cluster(backend="local", n_processes=n_proc, single_thread=False) sleep(30) else: print("multiprocessing disabled") dview = None n_processes = 1 print(n_processes) print("starting motion correction") sys.stdout.flush() mc = MotionCorrect(fnames, dview=dview, **opts.get_group("motion")) mc.motion_correct(save_movie=True) pw_rigid = mc_parameters["pw_rigid"] fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig if pw_rigid: bord_px = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) else: bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) border_nan = mc_parameters["border_nan"] bord_px = 0 if border_nan == "copy" else bord_px _ = cm.save_memmap(fname_mc, base_name="memmap_", order="C", border_to_0=bord_px) # if motion correction was performed, save the file # we save as hdf5 for better reading performance # downstream if motion_correct: print("saving motion corrected file") results_filebase = os.path.join(output_directory, job_name) mcorr_fname = results_filebase + "_mcorr.hdf5" dataset_name = opts.data["var_name_hdf5"] # fnames = opts.data["fnames"] # memmap_files = [] # for f in fnames: # if isinstance(f, bytes): # f = f.decode() # base_file = os.path.splitext(f)[0] # if pw_rigid: # memmap_pattern = base_file + "*_els_*" # else: # memmap_pattern = base_file + "*_rig_*" # memmap_files += sorted(glob.glob(memmap_pattern)) if pw_rigid: memmap_files = mc.fname_tot_els else: memmap_files = mc.fname_tot_rig # get the frame shape mov = cm.load(memmap_files[0]) frame_shape = mov.shape[-2::] write_hdf5_movie( movie_name=mcorr_fname, memmap_files=memmap_files, frame_shape=frame_shape, dataset_name=dataset_name, compression="gzip", ) # save the parameters in the same dir as the results final_params = opts.to_dict() params_file = os.path.join(output_directory, "all_mcorr_parameters.pkl") with open(params_file, "wb") as fp: pickle.dump(final_params, fp) # deleting mcorr unused memmap files if pw_rigid: memmap_files = mc.fname_tot_els else: memmap_files = mc.fname_tot_rig print(f"deleting {memmap_files}") for f in memmap_files: os.remove(f) print("stopping server") cm.stop_server(dview=dview)
def main(): pass # For compatibility between running under Spyder and the CLI # %% start a cluster c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% set up some parameters fnames = [ os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif') ] # file(s) to be analyzed is_patches = True # flag for processing in patches or not fr = 10 # approximate frame rate of data decay_time = 5.0 # length of transient if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 10 # half size of each patch stride = 4 # overlap between patches K = 4 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = 30 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order params_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'rf': rf, 'stride': stride, 'K': K, 'gSig': gSig, 'merge_thr': merge_thresh, 'p': p, 'nb': gnb } opts = params.CNMFParams(params_dict=params_dict) # %% Now RUN CaImAn Batch (CNMF) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit_file() # %% plot contour plots of components Cns = local_correlations_movie_offline(fnames[0], remove_baseline=True, swap_dim=False, window=1000, stride=1000, winSize_baseline=100, quantil_min_baseline=10, dview=dview) Cn = Cns.max(axis=0) cnm.estimates.plot_contours(img=Cn) # %% load memory mapped file Yr, dims, T = cm.load_memmap(cnm.mmap_file) images = np.reshape(Yr.T, [T] + list(dims), order='F') # %% refit cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) min_SNR = 2 # peak SNR for accepted components (if above this, acept) rval_thr = 0.85 # space correlation threshold (if above this, accept) use_cnn = True # use the CNN classifier min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': use_cnn, 'min_cnn_thr': min_cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% visualize selected and rejected components cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% visualize selected components cnm2.estimates.view_components(images, idx=cnm2.estimates.idx_components, img=Cn) #%% only select high quality components (destructive) # cnm2.estimates.select_components(use_object=True) # cnm2.estimates.plot_contours(img=Cn) #%% save results cnm2.estimates.Cn = Cn cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') # %% play movie with results (original, reconstructed, amplified residual) cnm2.estimates.play_movie(images, magnification=4) # %% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file)
def run_source_extraction(row, parameters, dview, session_wise = False): ''' This is the function for source extraction. Its goal is to take in a .mmap file, perform source extraction on it using cnmf-e and save the cnmf object as a .pkl file. Args: row: pd.DataFrame object The row corresponding to the analysis state to be source extracted. Returns: row: pd.DataFrame object The row corresponding to the source extracted analysis state. ''' step_index = 5 row_local = row.copy() row_local.loc['source_extraction_parameters'] = str(parameters) row_local = db.set_version_analysis('source_extraction',row_local,session_wise) index = row_local.name # Determine input path if parameters['session_wise']: input_mmap_file_path = eval(row_local.loc['alignment_output'])['main'] if parameters['equalization']: input_mmap_file_path =eval(row_local['equalization_output'])['main'] else: input_mmap_file_path = eval(row_local.loc['motion_correction_output'])['main'] if not os.path.isfile(input_mmap_file_path): logging.error('Input file does not exist. Cancelling.') return row_local # Determine output paths file_name = db.create_file_name(step_index, index) if parameters['session_wise']: data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/session_wise/' else: data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/trial_wise/' output_file_path = data_dir + f'main/{file_name}.hdf5' # Create a dictionary with parameters output = { 'main': output_file_path, 'meta':{ 'analysis' : { 'analyst' : os.environ['ANALYST'], 'date' : datetime.datetime.today().strftime("%m-%d-%Y"), 'time' : datetime.datetime.today().strftime("%H:%M:%S"), }, 'duration': {} } } # Load memmory mappable input file if os.path.isfile(input_mmap_file_path): Yr, dims, T = cm.load_memmap(input_mmap_file_path) # logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.') images = Yr.T.reshape((T,) + dims, order='F') else: logging.warning(f'{index} .mmap file does not exist. Cancelling') return row_local # SOURCE EXTRACTION # Check if the summary images are already there corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(index, gSig_abs = parameters['gSig'][0]) if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path): # Already computed summary images logging.info(f'{index} Already computed summary images') cn_filter = np.load(corr_npy_file_path) pnr = np.load(pnr_npy_file_path) else: # Compute summary images t0 = datetime.datetime.today() logging.info(f'{index} Computing summary images') cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig = parameters['gSig'][0], swap_dim=False) dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes output['meta']['duration']['summary_images'] = dt logging.info(f'{index} Computed summary images. dt = {dt} min') # Saving summary images as npy files gSig = parameters['gSig'][0] corr_npy_file_path = data_dir + f'/meta/corr/{db.create_file_name(3, index)}_gSig_{gSig}.npy' pnr_npy_file_path = data_dir + f'/meta/pnr/{db.create_file_name(3, index)}_gSig_{gSig}.npy' with open(corr_npy_file_path, 'wb') as f: np.save(f, cn_filter) with open(pnr_npy_file_path, 'wb') as f: np.save(f, pnr) # Store the paths in the meta dictionary output['meta']['corr'] = {'main': corr_npy_file_path, 'meta': {}} output['meta']['pnr'] = {'main': pnr_npy_file_path, 'meta': {}} # Calculate min, mean, max value for cn_filter and pnr corr_min, corr_mean, corr_max = cn_filter.min(), cn_filter.mean(), cn_filter.max() output['meta']['corr']['meta'] = {'min': corr_min, 'mean': corr_mean, 'max': corr_max} pnr_min, pnr_mean, pnr_max = pnr.min(), pnr.mean(), pnr.max() output['meta']['pnr']['meta'] = {'min': pnr_min, 'mean': pnr_mean, 'max': pnr_max} # If min_corr and min_pnr are specified via a linear equation, calculate # this value if type(parameters['min_corr']) == list: min_corr = parameters['min_corr'][0]*corr_mean + parameters['min_corr'][1] parameters['min_corr'] = min_corr logging.info(f'{index} Automatically setting min_corr = {min_corr}') if type(parameters['min_pnr']) == list: min_pnr = parameters['min_pnr'][0]*pnr_mean + parameters['min_pnr'][1] parameters['min_pnr'] = min_pnr logging.info(f'{index} Automatically setting min_pnr = {min_pnr}') # Set the parameters for caiman opts = params.CNMFParams(params_dict = parameters) # SOURCE EXTRACTION logging.info(f'{index} Performing source extraction') t0 = datetime.datetime.today() n_processes = psutil.cpu_count() logging.info(f'{index} n_processes: {n_processes}') cnm = cnmf.CNMF(n_processes = n_processes, dview = dview, params = opts) cnm.fit(images) cnm.estimates.dims = dims # Store the number of neurons output['meta']['K'] = len(cnm.estimates.C) # Calculate the center of masses cnm.estimates.center = caiman.base.rois.com(cnm.estimates.A, images.shape[1], images.shape[2]) # Save the cnmf object as a hdf5 file logging.info(f'{index} Saving cnmf object') cnm.save(output_file_path) dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes output['meta']['duration']['source_extraction'] = dt logging.info(f'{index} Source extraction finished. dt = {dt} min') # Write necessary variables in row and return row_local.loc['source_extraction_parameters'] = str(parameters) row_local.loc['source_extraction_output'] = str(output) return row_local
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) # fnames = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.nwb')] fnames = [ os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.nwb') ] # filename to be created or processed # dataset dependent parameters fr = 5 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds starting_time = 0. #%% load the file and save it in the NWB format (if it doesn't exist already) if not os.path.exists(fnames[0]): # fnames_orig = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.h5')] # filename to be processed fnames_orig = [ os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.h5') ] # filename to be processed orig_movie = cm.load(fnames_orig, fr=fr, is3D=True) # orig_movie = cm.load_movie_chain(fnames_orig,fr=fr,is3D=True) # save file in NWB format with various additional info orig_movie.save(fnames[0], sess_desc='test', identifier='demo 3d', exp_desc='demo movie', imaging_plane_description='multi plane', emission_lambda=520.0, indicator='none', location='visual cortex', starting_time=starting_time, experimenter='NAOMi', lab_name='Tank Lab', institution='Princeton U', experiment_description='Experiment Description', session_id='Session 1', var_name_hdf5='TwoPhotonSeries') #%% First setup some parameters for data and motion correction # motion correction parameters dxy = (1., 1., 5.) # spatial resolution in x, y, and z in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (10., 10., 10.) # maximum shift in um patch_motion_um = (50., 50., 30. ) # patch size for non-rigid correction in um # pw_rigid = False # flag to select rigid vs pw_rigid motion correction niter_rig = 1 pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a / b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24, 4) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 is3D = True mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'niter_rig': niter_rig, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy', 'var_name_hdf5': 'acquisition/TwoPhotonSeries', 'is3D': is3D, 'splits_els': 12, 'splits_rig': 12 } opts = params.CNMFParams( params_dict=mc_dict ) #NOTE: default adjustments of parameters are not set yet, manually setting them now # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames, var_name_hdf5=opts.data['var_name_hdf5'], is3D=True) T, h, w, z = m_orig.shape # Time, plane, height, weight m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w)) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing # NOTE: ignore dview right now for debugging purposes # c, dview, n_processes = cm.cluster.setup_cluster( # backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=None, var_name_hdf5=opts.data['var_name_hdf5'], **opts.get_group('motion')) # mc = MotionCorrect(fnames, dview=dview, var_name_hdf5=opts.data['var_name_hdf5'], **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames, var_name_hdf5=opts.data['var_name_hdf5'], is3D=True) T, h, w, z = m_orig.shape # Time, plane, height, weight m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w)) m_els = cm.load(mc.mmap_file, is3D=True) m_els = np.reshape(np.transpose(m_els, (3, 0, 1, 2)), (T * z, h, w)) ds_ratio = 0.2 moviehandle = cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) root = '/Users/hheiser/Desktop/testing data/chronic_M2N3/0d_baseline/channel1' fnames = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\1\file_00003.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\2\file_00004.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\3\file_00005.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\4\file_00006.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\5\file_00007.tif', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\6\file_00008.tif'] fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) #%% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=30, magnification=1, do_loop=False) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory #%% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) #%% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames[:3]) m_els = cm.load(mmap_file[:3]) ds_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie, m_els.resize(1, 1, ds_ratio)], axis=2) moviehandle.play(fr=15, q_max=99.5, magnification=2) # press q to exit #%% MEMORY MAPPING border_to_0 = 0 if mc.border_nan == 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries mmap_file = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\1\file_00001_els__d1_512_d2_512_d3_1_order_F_frames_1147_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\2\file_00002_els__d1_512_d2_512_d3_1_order_F_frames_2520_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\3\file_00003_els__d1_512_d2_512_d3_1_order_F_frames_3814_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\4\file_00004_els__d1_512_d2_512_d3_1_order_F_frames_5154_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\5\file_00005_els__d1_512_d2_512_d3_1_order_F_frames_2677_.mmap', r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\6\file_00006_els__d1_512_d2_512_d3_1_order_F_frames_3685_.mmap'] fname_new = r'E:\PhD\Data\DG\M14_20191014\N1\memmap__d1_512_d2_512_d3_1_order_C_frames_34939_.mmap' # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = {'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub} opts.change_params(params_dict=opts_dict) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) #opts.change_params({'p': 1,'rf':None, 'only_init':False}) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) #%% RUN CNMF SEEDED WITH MANUAL MASK # load mask mask = np.asarray(imageio.imread('/Users/hheiser/Desktop/testing data/file_00020_no_motion/avg_mask_fixed.png'), dtype=bool) # get component ROIs from the mask and plot them Ain, labels, mR = cm.base.rois.extract_binary_masks(mask) # plot original mask and extracted labels to check mask fig, ax = plt.subplots(1,2) ax[0].imshow(-mR,cmap='binary') ax[0].set_title('Original mask') ax[1].imshow(labels) ax[1].set_title('Extracted labelled ROIs') """" plt.figure() crd = cm.utils.visualization.plot_contours( Ain.astype('float32'), mR, thr=0.99, display_numbers=True) # todo check if this is important for the pipeline plt.title('Contour plots of detected ROIs in the structural channel') """ opts.change_params({'rf': None, 'only_init': False}) # run CNMF seeded with this mask cnm_corr = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain) cnm_corr_seed = cnm_corr_seed.fit(images) #cnm_seed = cnm_seed.fit_file(motion_correct=False) #%% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) #%% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.evaluate_components(images, params=cnm2.params, dview=dview) #%% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) #%% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components #### -> will delete rejected components! cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=1, bpx=border_to_0, include_bck=True) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% save results dirname = fnames[0][:-4] + "_results.hdf5" cnm2.estimates.Cn = Cn cnm2.save(dirname) #load results cnm2 = cnmf.load_CNMF(dirname) mov_name = fnames[0][:-4] + "_movie_restored_2_gain.tif" helper.save_movie(cnm2.estimates,images,mov_name,frame_range=range(200),include_bck=True)
def run_alignmnet(selected_rows, parameters, dview): ''' This is the main function for the alignment step. It applies methods from the CaImAn package used originally in motion correction to do alignment. Args: df: pd.DataFrame A dataframe containing the analysis states you want to have aligned. parameters: dict The alignment parameters. dview: object The dview object Returns: df: pd.DataFrame A dataframe containing the aligned analysis states. ''' # Sort the dataframe correctly df = selected_rows.copy() df = df.sort_values(by=paths.multi_index_structure) # Determine the mouse and session of the dataset index = df.iloc[0].name mouse, session, *r = index # alignment_v = index[len(paths.data_structure) + step_index] alignment_v = len(df) alignment_index = (mouse, session, alignment_v) # Determine the output .mmap file name file_name = f'mouse_{mouse}_session_{session}_v{alignment_v}' output_mmap_file_path = os.environ['DATA_DIR'] + f'data/interim/alignment/main/{file_name}.mmap' try: df.reset_index()[['session','trial', 'is_rest']].set_index(['session','trial', 'is_rest'], verify_integrity=True) except ValueError: logging.error('You passed multiple of the same trial in the dataframe df') return df output = { 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S") }, 'duration': {} } } # Get necessary parameters motion_correction_parameters_list = [] motion_correction_output_list = [] input_mmap_file_list = [] trial_index_list = [] x_ = [] _x = [] y_ = [] _y = [] for idx, row in df.iterrows(): motion_correction_parameters_list.append(eval(row.loc['motion_correction_parameters'])) motion_correction_output = eval(row.loc['motion_correction_output']) motion_correction_output_list.append(motion_correction_output) input_mmap_file_list.append(motion_correction_output['main']) trial_index_list.append(db.get_trial_name(idx[2], idx[3])) [x1,x2,y1,y2] = motion_correction_output['meta']['cropping_points'] x_.append(x1) _x.append(x2) y_.append(y1) _y.append(y2) new_x1 = max(x_) new_x2 = max(_x) new_y1 = max(y_) new_y2 = max(_y) m_list = [] for i in range(len(input_mmap_file_list)): m = cm.load(input_mmap_file_list[i]) motion_correction_output = eval(df.iloc[i].loc['motion_correction_output']) [x1,x2,y1,y2] = motion_correction_output['meta']['cropping_points'] m = m.crop(new_x1 - x1, new_x2 - x2, new_y1 - y1, new_y2 - y2, 0, 0) m_list.append(m) # Concatenate them using the concat function m_concat = cm.concatenate(m_list, axis=0) data_dir = os.environ['DATA_DIR'] + 'data/interim/alignment/main/' file_name = db.create_file_name(step_index, index) fname= m_concat.save(data_dir + file_name + '.mmap', order='C') #meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] #output['meta']['cropping_points'] = [x_, _x, y_, _y] # Save the movie #fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') #logging.info(f'{index} Cropped and saved rigid movie as {fname_tot_els}') # MOTION CORRECTING EACH INDIVIDUAL MOVIE WITH RESPECT TO A TEMPLATE MADE OF THE FIRST MOVIE logging.info(f'{alignment_index} Performing motion correction on all movies with respect to a template made of \ the first movie.') t0 = datetime.datetime.today() # Create a template of the first movie template_index = trial_index_list.index(parameters['make_template_from_trial']) m0 = cm.load(input_mmap_file_list[template_index ]) [x1, x2, y1, y2] = motion_correction_output_list[template_index]['meta']['cropping_points'] m0 = m0.crop(new_x1 - x1, new_x2 - x2, new_y1 - y1, new_y2 - y2, 0, 0) m0_filt = cm.movie( np.array([high_pass_filter_space(m_, parameters['gSig_filt']) for m_ in m0])) template0 = cm.motion_correction.bin_median( m0_filt.motion_correct(5, 5, template=None)[0]) # may be improved in the future # Setting the parameters opts = params.CNMFParams(params_dict=parameters) # Create a motion correction object mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion')) # Perform non-rigid motion correction mc.motion_correct(template=template0, save_movie=True) # Cropping borders x_ = math.ceil(abs(np.array(mc.shifts_rig)[:, 1].max()) if np.array(mc.shifts_rig)[:, 1].max() > 0 else 0) _x = math.ceil(abs(np.array(mc.shifts_rig)[:, 1].min()) if np.array(mc.shifts_rig)[:, 1].min() < 0 else 0) y_ = math.ceil(abs(np.array(mc.shifts_rig)[:, 0].max()) if np.array(mc.shifts_rig)[:, 0].max() > 0 else 0) _y = math.ceil(abs(np.array(mc.shifts_rig)[:, 0].min()) if np.array(mc.shifts_rig)[:, 0].min() < 0 else 0) # Load the motion corrected movie into memory movie= cm.load(mc.fname_tot_rig[0]) # Crop all movies to those border pixels movie.crop(x_, _x, y_, _y, 0, 0) output['meta']['cropping_points'] = [x_, _x, y_, _y] #save motion corrected and cropped movie output_mmap_file_path_tot = movie.save(data_dir + file_name + '.mmap', order='C') logging.info(f'{index} Cropped and saved rigid movie as {output_mmap_file_path_tot}') # Save the path in teh output dictionary output['main'] = output_mmap_file_path_tot # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) # Create a timeline and store it timeline = [[trial_index_list[0], 0]] timepoints = [0] for i in range(1, len(m_list)): m = m_list[i] timeline.append([trial_index_list[i], timeline[i - 1][1] + m.shape[0]]) timepoints.append(timepoints[i-1]+ m.shape[0]) timeline_pkl_file_path = os.environ['DATA_DIR'] + f'data/interim/alignment/meta/timeline/{file_name}.pkl' with open(timeline_pkl_file_path,'wb') as f: pickle.dump(timeline,f) output['meta']['timeline'] = timeline_pkl_file_path timepoints.append(movie.shape[0]) dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes output['meta']['duration']['concatenation'] = dt logging.info(f'{alignment_index} Performed concatenation. dt = {dt} min.') for idx, row in df.iterrows(): df.loc[idx, 'alignment_output'] = str(output) df.loc[idx, 'alignment_parameters'] = str(parameters) ## modify all motion correction file to the aligned version data_dir = os.environ['DATA_DIR'] + 'data/interim/motion_correction/main/' for i in range(len(input_mmap_file_list)): row = df.iloc[i].copy() motion_correction_output_list.append(motion_correction_output) aligned_movie = movie[timepoints[i]:timepoints[i+1]] file_name = db.create_file_name(2, selected_rows.iloc[i].name) motion_correction_output_aligned = aligned_movie.save(data_dir + file_name + '_els' + '.mmap', order='C') new_output= {'main' : motion_correction_output_aligned } new_dict = eval(row['motion_correction_output']) new_dict.update(new_output) row['motion_correction_output'] = str(new_dict) df = db.append_to_or_merge_with_states_df(df, row) # # Delete the motion corrected movies # for fname in mc.fname_tot_rig: # os.remove(fname) return df
None, # half-size of patch in pixels. If None, no patches are constructed and the whole FOV is processed jointly #'stride': stride, 'update_num_comps': False, 'motion_correct': False, 'sniper_mode': True, # whether to use the online CNN classifier for screening candidate components (otherwise space correlation is used) 'thresh_CNN_noisy': thresh_CNN_noisy, 'K': K, 'expected_comps': K, 'update_num_comps': False, # whether to search for new components 'min_num_trial': new_K, 'method_deconvolution': deconv_method, 'show_movie': True } allParams = params.CNMFParams(params_dict=initialParamsDict ) # define parameters in the params.CNMFParams caimanResults = cnmf.online_cnmf.OnACID( params=allParams) # pass parameters to caiman object timer.toc() # %% ********* Wait for initialization trigger message from MicroManager: ********* print("Now waiting for MicroManager to capture " + str(initFrames) + " initialization frames..") print("*** Starting Initialization protocol with " + initMethod_online + " method ***") caimanResults.initialize_online() # initialize model timer.toc() # %% ********* Visualize results of initialization: ********* print(
def main(): pass # For compatibility between running under Spyder and the CLI #%% Select file(s) to be processed (download if not present) fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fnames = [download_demo(fnames[0])] #%% First setup some parameters for data and motion correction # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = True if display_images: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # %% compare with original movie if display_images: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie, m_els.resize(1, 1, ds_ratio)], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 2 # spatial subsampling during initialization tsub = 2 # temporal subsampling during intialization # parameters for component evaluation opts_dict = {'fnames': fnames, 'p': p, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub} opts.change_params(params_dict=opts_dict); # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0). If you want to have # deconvolution within each patch change params.patch['p_patch'] to a # nonzero value #opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cns = local_correlations_movie_offline(mc.mmap_file[0], remove_baseline=True, window=1000, stride=1000, winSize_baseline=100, quantil_min_baseline=10, dview=dview) Cn = Cns.max(axis=0) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% save results cnm.estimates.Cn = Cn cnm.save(fname_new[:-5]+'_init.hdf5') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% VIEW TRACES (accepted and rejected) if display_images: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) #%% update object with selected components cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) #%% cnm2.estimates.Cn = Cn cnm2.save(cnm2.mmap_file[:-4] + 'hdf5') #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
opts = params.CNMFParams( params_dict={ 'fr': frate, 'dims': dims, 'decay_time': decay_time, 'method_init': 'corr_pnr', # use this for 1 photon 'K': K, 'gSig': gSig, 'gSiz': gSiz, 'merge_thr': merge_thr, 'p': p, 'tsub': tsub, 'ssub': ssub, 'rf': rf, 'stride': stride_cnmf, 'only_init': True, # set it to True to run CNMF-E 'nb': gnb, 'nb_patch': nb_patch, 'method_deconvolution': 'oasis', # could use 'cvxpy' alternatively 'low_rank_background': low_rank_background, 'update_background_components': True, # sometimes setting to False improve the results 'min_corr': min_corr, 'min_pnr': min_pnr, 'normalize_init': False, # just leave as is 'center_psf': True, # leave as is for 1 photon 'ssub_B': ssub_B, 'ring_size_factor': ring_size_factor, 'del_duplicates': True, # whether to remove duplicates from initialization 'border_pix': 0 # number of pixels to not consider in the borders) })
def extract_components(self, images, fname) -> Tuple[cnmf.CNMF, cnmf.CNMF, dict]: """ Uses constrained NNMF to extract spatial and temporal components, performs deconvolution and validates extracted components :param images: The tyx stack from which components should be extracted :param fname: The name of the original file from which <images> originated :return: [0]: Cnmf object after initial extraction [1]: Cnmf object after deconvolution and subsequent component validation [2]: Wrapped CaImAn parameter dictionary """ resolution = self.fov_um / images.shape[1] fr = 1 / self.time_per_frame # frame-rate dxy = (resolution, resolution) # spatial resolution in um/pixel c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) try: # set up parameters for source extraction p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed # TODO: Extraction fails if patch size is too small relative to neuron size (K<1). # Add intelligent computation of patch size # size patch to roughly have an edge length of 8 cells n_rad_pixels = self.neuron_radius / resolution rf = int(n_rad_pixels * 8) # half-size of the patches in pixels stride_cnmf = 6 # amount of overlap between the patches in pixels patch_area_um = ( rf * 2 * dxy[0] )**2 # we use this to calculate expected number of components per patch neur_area_um = np.pi * self.neuron_radius**2 K = int(patch_area_um / neur_area_um) # number of components (~neurons) per patch # expected half size of neurons in pixels gSig = [ int(self.neuron_radius / dxy[0]), int(self.neuron_radius / dxy[1]) ] method_init = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') ssub = 2 # spatial subsampling during initialization tsub = 1 # temporal subsampling during intialization # parameters for component evaluation opts_dict = { 'fnames': [ fname ], # NOTE: This parameter seems only necessary to allow contour extraction 'fr': fr, 'decay_time': self.decay_time, 'dxy': dxy, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': int(n_processes ), # return type is numpy.int64 which can't json serialize 'only_init': True, 'ssub': ssub, 'tsub': tsub } opts = params.CNMFParams(params_dict=opts_dict) # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) # rerun seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # Validate components val_dict = { 'decay_time': self.decay_time, 'min_SNR': self.min_snr, 'SNR_lowest': self.snr_lowest, 'rval_thr': self.rval_thr, 'rval_lowest': self.rval_lowest, 'use_cnn': self.use_cnn, 'min_cnn_thr': self.cnn_thr, 'cnn_lowest': self.cnn_lowest } cnm2.params.set('quality', val_dict) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # update object with selected components cnm2.estimates.select_components(use_object=True) # extract DF/F values cnm2.estimates.detrend_df_f(**self.detrend_dff_params) finally: cm.stop_server(dview=dview) return cnm, cnm2, {"CNMF": opts_dict, "Validation": val_dict}
def motion_correct( self, fname: str, co_fname: str) -> Tuple[np.ndarray, dict, Optional[np.ndarray]]: """ Uses caiman non-rigid motion correction to remove/reduce motion artefacts Note: ALways saves an intermediate mem-map representation in order C of the corrected 32-bit stack :param fname: The filename of the source file :param co_fname: Filename of a stack that should be co-aligned or None :return: [0]: Corrected stack as a memmap [1]: Wrapped CaImAn parameter dictionary [2]: Corrected co-stack as a memmap or None """ cont_folder = path.dirname(fname) # the containing folder stack_name = path.split(fname)[1] save_dir = cont_folder + f"/{self.ana_dir}" if not path.exists(save_dir): makedirs(save_dir) print("Created analysis directory", flush=True) out_name = save_dir + '/' + stack_name if co_fname is not None: co_stack_name = path.split(co_fname)[1] co_out_name = save_dir + '/' + co_stack_name else: co_out_name = None test_image = imread( fname, key=0) # load first frame of stack to compute resolution assert test_image.shape[0] == test_image.shape[1] resolution = self.fov_um / test_image.shape[0] fr = 1 / self.time_per_frame # frame-rate dxy = (resolution, resolution) # spatial resolution in um/pixel max_shift_um = (self.neuron_radius * 4, self.neuron_radius * 4 ) # maximally allow shift by ~2 cell diameters # maximum allowed rigid shift in pixels max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)] # start a new patch for pw-rigid motion correction every x pixels strides = tuple( [int(a / b) for a, b in zip(self.patch_motion_um, dxy)]) # overlap between patches (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts (unit unclear - likely pixels as it is int) max_deviation_rigid = 3 # create parameter dictionary mc_dict = { 'fnames': [fname], 'fr': fr, 'decay_time': self.decay_time, 'dxy': dxy, 'pw_rigid': self.pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) try: # motion correction mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion')) # Run (piecewise-rigid motion) correction using NoRMCorre # if we don't save the movie into a memmap, there doesn't seem to be # any possibility to get at the corrected data later??? mc.motion_correct(save_movie=True) # memory mapping border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name=out_name, order='C', border_to_0=border_to_0) # delete the original mem-map if mc.fname_tot_els is None: [remove(fn) for fn in mc.fname_tot_rig] else: [remove(fn) for fn in mc.fname_tot_els] # now load the new file and transform output into [z,y,x] stack yr, dims, n_t = cm.load_memmap(fname_new) images = np.reshape(yr.T, [n_t] + list(dims), order='F') if self.save_projection: # save anatomical projection as 16bit tif anat_projection = images.copy() anat_projection = np.sum(anat_projection, 0) anat_projection -= np.min(anat_projection) anat_projection /= np.max(anat_projection) anat_projection *= (2**16 - 1) anat_projection[anat_projection < 0] = 0 anat_projection[anat_projection > (2**16 - 1)] = (2**16 - 1) anat_projection = anat_projection.astype(np.uint16) imsave(out_name, anat_projection, imagej=True, resolution=(1 / dxy[0], 1 / dxy[1]), metadata={ 'axes': 'YX', 'unit': 'um' }) # Repeat for the co-stack if present if co_fname is not None: co_aligned_images = np.array(mc.apply_shifts_movie(co_fname)) if self.save_projection: # save anatomical projection as 16bit tif anat_projection = np.sum(co_aligned_images, 0) anat_projection -= np.min(anat_projection) anat_projection /= np.max(anat_projection) anat_projection *= (2**16 - 1) anat_projection[anat_projection < 0] = 0 anat_projection[anat_projection > (2**16 - 1)] = (2**16 - 1) anat_projection = anat_projection.astype(np.uint16) imsave(co_out_name, anat_projection, imagej=True, resolution=(1 / dxy[0], 1 / dxy[1]), metadata={ 'axes': 'YX', 'unit': 'um' }) else: co_aligned_images = None finally: cm.stop_server(dview=dview) return images, {"Motion Correction": mc_dict}, co_aligned_images
def main(): pass # For compatibility between running under Spyder and the CLI # %% start the cluster try: cm.stop_server() # stop it if it was running except (): pass c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes= 24, # number of process to use, if you go out of memory try to reduce this one single_thread=False) # %% First setup some parameters for motion correction # dataset dependent parameters fnames = ['data_endoscope.tif'] # filename to be processed fnames = [download_demo(fnames[0])] # download file if not already present filename_reorder = fnames fr = 10 # movie frame rate decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters motion_correct = True # flag for motion correction pw_rigid = False # flag for pw-rigid motion correction gSig_filt = (3, 3) # size of filter, in general gSig (see below), # change this one if algorithm does not work max_shifts = (5, 5) # maximum allowed rigid shift strides = ( 48, 48 ) # start a new patch for pw-rigid motion correction every x pixels overlaps = (24, 24 ) # overlap between pathes (size of patch strides+overlaps) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 border_nan = 'copy' mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'gSig_filt': gSig_filt, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': border_nan } opts = params.CNMFParams(params_dict=mc_dict) # %% MOTION CORRECTION # The pw_rigid flag set above, determines where to use rigid or pw-rigid # motion correction if motion_correct: # do motion correction rigid mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) mc.motion_correct(save_movie=True) fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig if pw_rigid: bord_px = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) else: bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) plt.subplot(1, 2, 1) plt.imshow(mc.total_template_rig) # % plot template plt.subplot(1, 2, 2) plt.plot(mc.shifts_rig) # % plot rigid shifts plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames') plt.ylabel('pixels') bord_px = 0 if border_nan == 'copy' else bord_px fname_new = cm.save_memmap(fname_mc, base_name='memmap_', order='C', border_to_0=bord_px) else: # if no motion correction just memory map the file fname_new = cm.save_memmap(filename_reorder, base_name='memmap_', order='C', border_to_0=0, dview=dview) # load memory mappable file Yr, dims, T = cm.load_memmap(fname_new) images = Yr.T.reshape((T, ) + dims, order='F') # %% Parameters for source extraction and deconvolution (CNMF-E algorithm) p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None for 1p data gSig = ( 3, 3 ) # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1 Ain = None # possibility to seed with predetermined binary masks merge_thr = .7 # merging threshold, max correlation allowed rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80 stride_cnmf = 20 # amount of overlap between the patches in pixels # (keep it at least large as gSiz, i.e 4 times the neuron size gSig) tsub = 2 # downsampling factor in time for initialization, # increase if you have memory problems ssub = 1 # downsampling factor in space for initialization, # increase if you have memory problems # you can pass them here as boolean vectors low_rank_background = None # None leaves background of each patch intact, # True performs global low-rank approximation if gnb>0 gnb = 0 # number of background components (rank) if positive, # else exact ring model with following settings # gnb= 0: Return background as b and W # gnb=-1: Return full rank background B # gnb<-1: Don't return background nb_patch = 0 # number of background components (rank) per patch if gnb>0, # else it is set automatically min_corr = .8 # min peak value from correlation image min_pnr = 10 # min peak to noise ration from PNR image ssub_B = 2 # additional downsampling factor in space for background ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor opts.change_params( params_dict={ 'dims': dims, 'method_init': 'corr_pnr', # use this for 1 photon 'K': K, 'gSig': gSig, 'gSiz': gSiz, 'merge_thr': merge_thr, 'p': p, 'tsub': tsub, 'ssub': ssub, 'rf': rf, 'stride': stride_cnmf, 'only_init': True, # set it to True to run CNMF-E 'nb': gnb, 'nb_patch': nb_patch, 'method_deconvolution': 'oasis', # could use 'cvxpy' alternatively 'low_rank_background': low_rank_background, 'update_background_components': True, # sometimes setting to False improve the results 'min_corr': min_corr, 'min_pnr': min_pnr, 'normalize_init': False, # just leave as is 'center_psf': True, # leave as is for 1 photon 'ssub_B': ssub_B, 'ring_size_factor': ring_size_factor, 'del_duplicates': True, # whether to remove duplicates from initialization 'border_pix': bord_px }) # number of pixels to not consider in the borders) # %% compute some summary images (correlation and peak to noise) # change swap dim if output looks weird, it is a problem with tiffile cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig=gSig[0], swap_dim=False) # if your images file is too long this computation will take unnecessarily # long time and consume a lot of memory. Consider changing images[::1] to # images[::5] or something similar to compute on a subset of the data # inspect the summary images and set the parameters inspect_correlation_pnr(cn_filter, pnr) # print parameters set above, modify them if necessary based on summary images print(min_corr) # min correlation of peak (from correlation image) print(min_pnr) # min peak to noise ratio # %% RUN CNMF ON PATCHES cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts) cnm.fit(images) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% DISCARD LOW QUALITY COMPONENTS min_SNR = 2.5 # adaptive way to set threshold on the transient size r_values_min = 0.85 # threshold on space consistency (if you lower more components # will be accepted, potentially with worst quality) cnm.params.set('quality', { 'min_SNR': min_SNR, 'rval_thr': r_values_min, 'use_cnn': False }) cnm.estimates.evaluate_components(images, cnm.params, dview=dview) print(' ***** ') print('Number of total components: ', len(cnm.estimates.C)) print('Number of accepted components: ', len(cnm.estimates.idx_components)) # %% PLOT COMPONENTS cnm.dims = dims display_images = True # Set to true to show movies and images if display_images: cnm.estimates.plot_contours(img=cn_filter, idx=cnm.estimates.idx_components) cnm.estimates.view_components(images, idx=cnm.estimates.idx_components) # %% MOVIES display_images = False # Set to true to show movies and images if display_images: # fully reconstructed movie cnm.estimates.play_movie(images, q_max=99.5, magnification=2, include_bck=True, gain_res=10, bpx=bord_px) # movie without background cnm.estimates.play_movie(images, q_max=99.9, magnification=2, include_bck=False, gain_res=4, bpx=bord_px) # %% STOP SERVER cm.stop_server(dview=dview)
#'min_SNR': 8, #'SNR_lowest': 2.5, 'min_SNR': 6, 'SNR_lowest': 2.5, } max_thr = 0.45 vca1_neuron_sizes = {'max': 200, 'min': 10} dca1_neuron_sizes = { 'max': 110, #'min': 20 'min': 5 } neuron_size_params = vca1_neuron_sizes opts = params.CNMFParams(params_dict=eval_params) A = cnm_obj.estimates.A frames = session_trace_offset + range((vid_index - vid_start_index) * 1000, (vid_index - vid_start_index + 1) * 1000) images = video.load_images(local_mmap_fpath) if not filtered: cnm_obj.estimates.threshold_spatial_components(maxthr=max_thr) cnm_obj.estimates.remove_small_large_neurons( min_size_neuro=neuron_size_params['min'], max_size_neuro=neuron_size_params['max']) idx_components_bad = cnm_obj.estimates.idx_components_bad if idx_components_bad is None: idx_components_bad = [] if reevaluate:
def createParams(fnames, frameRate=20): # dataset dependent parameters #fr = 30 # imaging rate in frames per second # VIDEO #fr = 20 # imaging rate in frames per second # 2p #fr = 15 # imaging rate in frames per second fr = frameRate # first pass analysis was this #decay_time = 1.2 #0.4 # length of a typical transient in seconds # this is in folder 'slower' decay_time = 0.4 #2.0 #0.4 # length of a typical transient in seconds # VIDEO #myScaleFactor=5 # first run was 7 #myScaleFactor2 = 2 # 2P myScaleFactor = 1 # first run was 7 myScaleFactor2 = 1 # motion correction parameters strides = ( 48 * myScaleFactor, 48 * myScaleFactor ) #(48, 48) # start a new patch for pw-rigid motion correction every x pixels overlaps = ( 24 * myScaleFactor2, 24 * myScaleFactor2 ) #(24, 24) # overlap between pathes (size of patch strides+overlaps) max_shifts = (6 * myScaleFactor, 6 * myScaleFactor ) #(6,6) # maximum allowed rigid shifts (in pixels) max_deviation_rigid = 3 * myScaleFactor #3 # maximum shifts deviation allowed for patch with respect to rigid shifts pw_rigid = True # flag for performing non-rigid motion correction # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 * myScaleFactor #15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 * myScaleFactor #6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4 * myScaleFactor, 4 * myScaleFactor ] #[4, 4] # expected half size of neurons in pixels method_init = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') ssub = 1 # spatial subsampling during initialization tsub = 1 # temporal subsampling during intialization # parameters for component evaluation # VIDEO #min_SNR = 1.15 #2.0 # signal to noise ratio for accepting a component # 2P min_SNR = 2.0 # signal to noise ratio for accepting a component rval_thr = 0.85 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected print('============================================================') print('============================================================') print(' These are options manually set by cudmore') print( ' Be sure to double check them and be sure to decide on "video" or "2p"' ) print(' caimanOptions.createParams()') print(' fr:', fr) print(' decay_time:', decay_time) print(' myScaleFactor:', myScaleFactor) print(' myScaleFactor2:', myScaleFactor2) print(' min_SNR (for component evaluation):', min_SNR) print('============================================================') print('============================================================') ## ## opts_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'strides': strides, 'overlaps': overlaps, 'max_shifts': max_shifts, 'max_deviation_rigid': max_deviation_rigid, 'pw_rigid': pw_rigid, 'p': p, 'nb': gnb, 'rf': rf, 'K': K, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'only_init': True, 'ssub': ssub, 'tsub': tsub, 'merge_thr': merge_thr, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest } opts = params.CNMFParams(params_dict=opts_dict) return opts
def run_alignment(mouse, sessions, motion_correction_v, cropping_v, dview): """ This is the main function for the alignment step. It applies methods from the CaImAn package used originally in motion correction to do alignment. """ for session in sessions: # Update the database file_name = f"mouse_{mouse}_session_{session}_alignment" sql1 = "UPDATE Analysis SET alignment_main=? WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? " val1 = [file_name, mouse, session, motion_correction_v, cropping_v] cursor.execute(sql1, val1) # Determine the output .mmap file name output_mmap_file_path = os.environ[ 'DATA_DIR_LOCAL'] + f'data/interim/alignment/main/{file_name}.mmap' sql = "SELECT motion_correction_main FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? " val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() input_mmap_file_list = [] inter = [] for x in result: inter += x for y in inter: input_mmap_file_list.append(y) sql = "SELECT motion_correction_cropping_points_x1 FROM Analysis WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? " val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() x_ = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): x_.append(inter[j]) sql = "SELECT motion_correction_cropping_points_x2 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? " val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() _x = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): _x.append(inter[j]) sql = "SELECT motion_correction_cropping_points_y1 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() _y = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): _y.append(inter[j]) sql = "SELECT motion_correction_cropping_points_y2 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() y_ = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): y_.append(inter[j]) new_x1 = max(x_) new_x2 = max(_x) new_y1 = max(y_) new_y2 = max(_y) m_list = [] for i in range(len(input_mmap_file_list)): m = cm.load(input_mmap_file_list[i]) m = m.crop(new_x1 - x_[i], new_x2 - _x[i], new_y1 - y_[i], new_y2 - _y[i], 0, 0) m_list.append(m) # Concatenate them using the concat function m_concat = cm.concatenate(m_list, axis=0) fname = m_concat.save(output_mmap_file_path, order='C') # MOTION CORRECTING EACH INDIVIDUAL MOVIE WITH RESPECT TO A TEMPLATE MADE OF THE FIRST MOVIE logging.info( 'Performing motion correction on all movies with respect to a template made of the first movie.' ) t0 = datetime.datetime.today() # parameters alignment sql5 = "SELECT make_template_from_trial,gSig_filt,max_shifts,niter_rig,strides,overlaps,upsample_factor_grid,num_frames_split,max_deviation_rigid,shifts_opencv,use_conda,nonneg_movie, border_nan FROM Analysis WHERE alignment_main=? " val5 = [ file_name, ] cursor.execute(sql5, val5) myresult = cursor.fetchall() para = [] aux = [] for x in myresult: aux = x for y in aux: para.append(y) parameters = { 'make_template_from_trial': para[0], 'gSig_filt': (para[1], para[1]), 'max_shifts': (para[2], para[2]), 'niter_rig': para[3], 'strides': (para[4], para[4]), 'overlaps': (para[5], para[5]), 'upsample_factor_grid': para[6], 'num_frames_split': para[7], 'max_deviation_rigid': para[8], 'shifts_opencv': para[9], 'use_cuda': para[10], 'nonneg_movie': para[11], 'border_nan': para[12] } # Create a template of the first movie template_index = parameters['make_template_from_trial'] m0 = cm.load(input_mmap_file_list[1]) [x1, x2, y1, y2] = [x_, _x, y_, _y] for i in range(len(input_mmap_file_list)): m0 = m0.crop(new_x1 - x_[i], new_x2 - _x[i], new_y1 - y_[i], new_y2 - _y[i], 0, 0) m0_filt = cm.movie( np.array([ high_pass_filter_space(m_, parameters['gSig_filt']) for m_ in m0 ])) template0 = cm.motion_correction.bin_median( m0_filt.motion_correct( 5, 5, template=None)[0]) # may be improved in the future # Setting the parameters opts = params.CNMFParams(params_dict=parameters) # Create a motion correction object mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion')) # Perform non-rigid motion correction mc.motion_correct(template=template0, save_movie=True) # Cropping borders x_ = math.ceil( abs(np.array(mc.shifts_rig)[:, 1].max() ) if np.array(mc.shifts_rig)[:, 1].max() > 0 else 0) _x = math.ceil( abs(np.array(mc.shifts_rig)[:, 1].min() ) if np.array(mc.shifts_rig)[:, 1].min() < 0 else 0) y_ = math.ceil( abs(np.array(mc.shifts_rig)[:, 0].max() ) if np.array(mc.shifts_rig)[:, 0].max() > 0 else 0) _y = math.ceil( abs(np.array(mc.shifts_rig)[:, 0].min() ) if np.array(mc.shifts_rig)[:, 0].min() < 0 else 0) # Load the motion corrected movie into memory movie = cm.load(mc.fname_tot_rig[0]) # Crop all movies to those border pixels movie.crop(x_, _x, y_, _y, 0, 0) sql1 = "UPDATE Analysis SET alignment_x1=?, alignment_x2 =?, alignment_y1=?, alignment_y2=? WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val1 = [ x_, _x, y_, _y, mouse, session, motion_correction_v, cropping_v ] cursor.execute(sql1, val1) # save motion corrected and cropped movie output_mmap_file_path_tot = movie.save( os.environ['DATA_DIR_LOCAL'] + f'data/interim/alignment/main/{file_name}.mmap', order='C') logging.info( f' Cropped and saved rigid movie as {output_mmap_file_path_tot}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) # Create a timeline and store it sql = "SELECT trial FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() trial_index_list = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): trial_index_list.append(inter[j]) timeline = [[trial_index_list[0], 0]] timepoints = [0] for i in range(1, len(m_list)): m = m_list[i] timeline.append( [trial_index_list[i], timeline[i - 1][1] + m.shape[0]]) timepoints.append(timepoints[i - 1] + m.shape[0]) timeline_pkl_file_path = os.environ[ 'DATA_DIR'] + f'data/interim/alignment/meta/timeline/{file_name}.pkl' with open(timeline_pkl_file_path, 'wb') as f: pickle.dump(timeline, f) sql1 = "UPDATE Analysis SET alignment_timeline=? WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? " val1 = [ timeline_pkl_file_path, mouse, session, motion_correction_v, cropping_v ] cursor.execute(sql1, val1) timepoints.append(movie.shape[0]) dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes sql1 = "UPDATE Analysis SET alignment_duration_concatenation=? WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? " val1 = [dt, mouse, session, motion_correction_v, cropping_v] cursor.execute(sql1, val1) logging.info(f' Performed concatenation. dt = {dt} min.') ## modify all motion correction file to the aligned version data_dir = os.environ[ 'DATA_DIR'] + 'data/interim/motion_correction/main/' for i in range(len(input_mmap_file_list)): aligned_movie = movie[timepoints[i]:timepoints[i + 1]] motion_correction_output_aligned = aligned_movie.save( data_dir + file_name + '_els' + '.mmap', order='C') sql1 = "UPDATE Analysis SET motion_correct_align=? WHERE motion_correction_meta=? AND motion_correction_v" val1 = [ motion_correction_output_aligned, input_mmap_file_list[i], motion_correction_v ] cursor.execute(sql1, val1) database.commit() return
def main(): pass # For compatibility between running under Spyder and the CLI #%% """ General parameters """ play_movie = 1 plot_extras = 1 plot_extras_cell = 1 compute_mc_metrics = 1 #%% Select file(s) to be processed (download if not present) """ Load file """ #fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed #if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: # fnames = [download_demo(fnames[0])] #fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif'] #f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\' f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\' f_name = 'Ch1' f_ext = 'tif' fnames = [f_dir + f_name + '.' + f_ext] #fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5'] #%% First setup some parameters for data and motion correction """ Parameters """ # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 1 #0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) # note the lower than usual spatial resolution here max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid correction in um # motion correction parameters pw_rigid = True # flag to select rigid vs pw_rigid motion correction # maximum allowed rigid shift in pixels #max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)] max_shifts = [6, 6] # start a new patch for pw-rigid motion correction every x pixels #strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) strides = [48, 48] # overlap between pathes (size of patch in pixels: strides+overlaps) overlaps = (24, 24) # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'dxy': dxy, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': 'copy' } opts = params.CNMFParams(params_dict=mc_dict) # %% play the movie (optional) # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q if play_movie: m_orig = cm.load_movie_chain(fnames) ds_ratio = 0.2 moviehandle = m_orig.resize(1, 1, ds_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) # %% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the specified parameters mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) # note that the file is not loaded in memory # %% Run (piecewise-rigid motion) correction using NoRMCorre mc.motion_correct(save_movie=True) # type "mc."and press TAB to see all interesting associated variables and self. outputs # interesting outputs # saved file is mc.fname_tot_els / mc.fname_tot_rig # mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch # mc.coord_shifts_els: coordinates associated to patches with shifts # mc.total_template_els: updated template for pw # mc.total_template_rig: updated template for rigid # mc.templates_rig: templates for each iteration in rig #%% # compute metrics for the results (TAKES TIME!!) if compute_mc_metrics: # not finished bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) final_size = np.subtract( mc.total_template_els.shape, 2 * bord_px_els) # remove pixels in the boundaries winsize = 100 swap_dim = False resize_fact_flow = .2 # downsample for computing ROF tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction( fnames[0], final_size[0], final_size[1], swap_dim, winsize=winsize, play_flow=False, resize_fact_flow=resize_fact_flow) plt.figure() plt.plot(correlations_orig) # %% compare with original movie if play_movie: m_orig = cm.load_movie_chain(fnames) m_els = cm.load(mc.mmap_file) ds_ratio = 0.2 moviehandle = cm.concatenate([ m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie, m_els.resize(1, 1, ds_ratio) ], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit del m_orig del m_els if plot_extras: # plot total template plt.figure() plt.imshow(mc.total_template_els) plt.title('Template after iteration') # plot x and y corrections plt.figure() plt.plot(mc.shifts_rig) plt.title('Rigid motion correction xy movement') plt.legend(['x shift', 'y shift']) plt.xlabel('frames') # %% MEMORY MAPPING border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # you can include the boundaries of the FOV if you used the 'copy' option # during motion correction, although be careful about the components near # the boundaries # memory map the file in order 'C' fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% parameters for source extraction and deconvolution p = 2 # order of the autoregressive system gnb = 2 # number of global background components merge_thr = 0.85 # merging threshold, max correlation allowed rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 2 # number of components per patch gSig = [15, 15] # expected half size of neurons in pixels # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' ssub = 1 # spatial subsampling during initialization tsub = 1 # temporal subsampling during intialization # parameters for component evaluation opts_dict = { 'fnames': fnames, 'fr': fr, 'nb': gnb, 'rf': rf, 'K': K, 'gSig': gSig, 'stride': stride_cnmf, 'method_init': method_init, 'rolling_sum': True, 'merge_thr': merge_thr, 'n_processes': n_processes, 'only_init': True, 'ssub': ssub, 'tsub': tsub } opts.change_params(params_dict=opts_dict) # %% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) if plot_extras_cell: num_cell_plot = 51 plt.figure() plt.plot(cnm.estimates.C[num_cell_plot, :]) plt.title('Temporal component') plt.legend(['Cell ' + str(num_cell_plot)]) # plot component sptial profile A # first convert back to dense components plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape( list(dims)) plt.figure() plt.imshow(plot_spat_A) plt.title('Spatial component cell ' + str(num_cell_plot)) # %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE # you can also perform the motion correction plus cnmf fitting steps # simultaneously after defining your parameters object using # cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview) # cnm1.fit_file(motion_correct=True) # %% plot contours of found components Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') if plot_extras: plt.figure() plt.imshow(Cn) plt.title('Local correlations') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2 # signal to noise ratio for accepting a component rval_thr = 0.90 # space correlation threshold for accepting a component cnn_thr = 0.99 # threshold for CNN based classifier cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set( 'quality', { 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr, 'cnn_lowest': cnn_lowest }) cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% PLOT COMPONENTS cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Component selection: min_SNR=' + str(min_SNR) + '; rval_thr=' + str(rval_thr) + '; cnn prob range=[' + str(cnn_lowest) + ' ' + str(cnn_thr) + ']') # %% VIEW TRACES (accepted and rejected) if plot_extras: cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components) plt.suptitle('Accepted') cnm2.estimates.view_components(images, img=Cn, idx=cnm2.estimates.idx_components_bad) plt.suptitle('Rejected') #plt.figure(); #plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:]) # # # # #plt.figure(); #plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]); #plt.plot(); #plt.show(); # # #plt.figure(); #plt.plot(cnm2.estimates.detrend_df_f[1,:]) # these store the good and bad components, and next step sorts them # cnm2.estimates.idx_components # cnm2.estimates.idx_components_bad #%% update object with selected components #cnm2.estimates.select_components(use_object=True) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(img=Cn) plt.suptitle("Final results") #%% Save the mc data as in cmn struct as well ## #mc_out = dict( # pw_rigid = mc.pw_rigid, # fname = mc.fname, # mmap_file = mc.mmap_file, # total_template_els = mc.total_template_els, # total_template_rig = mc.total_template_rig, # border_nan = mc.border_nan, # border_to_0 = mc.border_to_0, # x_shifts_els = mc.x_shifts_els, # y_shifts_els = mc.y_shifts_els, # Cn = Cn # ) # # #deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out) #%% reconstruct denoised movie (press q to exit) if play_movie: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=False) # background not shown #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) save_results = True if save_results: cnm2.save(fnames[0][:-4] + '_results.hdf5')
def run_motion_correction(cropping_file, dview): """ This is the function for motion correction. Its goal is to take in a decoded and cropped .tif file, perform motion correction, and save the result as a .mmap file. This function is only runnable on the cn76 server because it requires parallel processing. Args: cropping_file: tif file after cropping dview: cluster Returns: row: pd.DataFrame object The row corresponding to the motion corrected analysis state. """ # Get output file paths data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/motion_correction/' sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,input,home_path,decoding_main FROM Analysis WHERE cropping_main=? ORDER BY motion_correction_v" val = [ cropping_file, ] cursor.execute(sql, val) result = cursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[6] == 0: data[6] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}" output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl' sql1 = "UPDATE Analysis SET motion_correction_meta=?,motion_correction_v=? WHERE cropping_main=? " val1 = [output_meta_pkl_file_path, data[6], cropping_file] cursor.execute(sql1, val1) else: data[6] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}" output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl' sql2 = "INSERT INTO Analysis (motion_correction_meta,motion_correction_v) VALUES (?,?)" val2 = [output_meta_pkl_file_path, data[6]] cursor.execute(sql2, val2) database.commit() sql3 = "UPDATE Analysis SET decoding_main=?,decoding_v=?,mouse=?,session=?,trial=?,is_rest=?,input=?,home_path=?,cropping_v=?,cropping_main=? WHERE motion_correction_meta=? AND motion_correction_v=?" val3 = [ data[9], data[4], data[0], data[1], data[2], data[3], data[7], data[8], data[5], cropping_file, output_meta_pkl_file_path, data[6] ] cursor.execute(sql3, val3) database.commit() output_meta_pkl_file_path_full = data_dir + output_meta_pkl_file_path # Calculate movie minimum to subtract from movie cropping_file_full = os.environ['DATA_DIR_LOCAL'] + cropping_file min_mov = np.min(cm.load(cropping_file_full)) # Apply the parameters to the CaImAn algorithm sql5 = "SELECT motion_correct,pw_rigid,save_movie_rig,gSig_filt,max_shifts,niter_rig,strides,overlaps,upsample_factor_grid,num_frames_split,max_deviation_rigid,shifts_opencv,use_conda,nonneg_movie, border_nan FROM Analysis WHERE cropping_main=? " val5 = [ cropping_file, ] cursor.execute(sql5, val5) myresult = cursor.fetchall() para = [] aux = [] for x in myresult: aux = x for y in aux: para.append(y) parameters = { 'motion_correct': para[0], 'pw_rigid': para[1], 'save_movie_rig': para[2], 'gSig_filt': (para[3], para[3]), 'max_shifts': (para[4], para[4]), 'niter_rig': para[5], 'strides': (para[6], para[6]), 'overlaps': (para[7], para[7]), 'upsample_factor_grid': para[8], 'num_frames_split': para[9], 'max_deviation_rigid': para[10], 'shifts_opencv': para[11], 'use_cuda': para[12], 'nonneg_movie': para[13], 'border_nan': para[14] } caiman_parameters = parameters.copy() caiman_parameters['min_mov'] = min_mov opts = params.CNMFParams(params_dict=caiman_parameters) # Rigid motion correction (in both cases) logging.info('Performing rigid motion correction') t0 = datetime.datetime.today() # Create a MotionCorrect object mc = MotionCorrect([cropping_file_full], dview=dview, **opts.get_group('motion')) # Perform rigid motion correction mc.motion_correct_rigid(save_movie=parameters['save_movie_rig'], template=None) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes logging.info(f' Rigid motion correction finished. dt = {dt} min') # Obtain template, rigid shifts and border pixels total_template_rig = mc.total_template_rig shifts_rig = mc.shifts_rig # Save template, rigid shifts and border pixels in a dictionary meta_pkl_dict = { 'rigid': { 'template': total_template_rig, 'shifts': shifts_rig, } } sql = "UPDATE Analysis SET duration_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [dt, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) if parameters['save_movie_rig'] == 1: # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f' Loading rigid movie for cropping') m_rig = cm.load(mc.fname_tot_rig[0]) logging.info(f' Loaded rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_rigid_shifts(shifts_rig) # Crop the movie logging.info( f' Cropping and saving rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_rig = m_rig.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['rigid']['cropping_points'] = [x_, _x, y_, _y] sql = "UPDATE Analysis SET motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [x_, _x, y_, _y, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) # Save the movie rig_role = 'alternate' if parameters['pw_rigid'] else 'main' fname_tot_rig = m_rig.save(data_dir + rig_role + '/' + file_name + '_rig' + '.mmap', order='C') logging.info(f' Cropped and saved rigid movie as {fname_tot_rig}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) sql = "UPDATE Analysis SET motion_correction_rig_role=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [fname_tot_rig, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) database.commit() # If specified in the parameters, apply piecewise-rigid motion correction if parameters['pw_rigid'] == 1: logging.info(f' Performing piecewise-rigid motion correction') t0 = datetime.datetime.today() # Perform non-rigid (piecewise rigid) motion correction. Use the rigid result as a template. mc.motion_correct_pwrigid(save_movie=True, template=total_template_rig) # Obtain template and filename total_template_els = mc.total_template_els fname_tot_els = mc.fname_tot_els[0] dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes meta_pkl_dict['pw_rigid'] = { 'template': total_template_els, 'x_shifts': mc.x_shifts_els, 'y_shifts': mc. y_shifts_els # removed them initially because they take up space probably } logging.info( f' Piecewise-rigid motion correction finished. dt = {dt} min') # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f' Loading pw-rigid movie for cropping') m_els = cm.load(fname_tot_els) logging.info(f' Loaded pw-rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_pw_rigid_shifts( np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) # Crop the movie logging.info( f' Cropping and saving pw-rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_els = m_els.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] # Save the movie fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') logging.info(f'Cropped and saved rigid movie as {fname_tot_els}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_els[0]) sql = "UPDATE Analysis SET motion_correction_main=?, motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=?,duration_pw_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [ fname_tot_els, x_, _x, y_, _y, dt, output_meta_pkl_file_path, data[6] ] cursor.execute(sql, val) database.commit() # Write meta results dictionary to the pkl file pkl_file = open(output_meta_pkl_file_path_full, 'wb') pickle.dump(meta_pkl_dict, pkl_file) pkl_file.close() return fname_tot_els, data[6]
def main(): pass # For compatibility between running under Spyder and the CLI c, dview, n_processes =\ cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% set up some parameters fnames = [os.path.join(caiman_datadir(), 'split', 'first3000-ch1.tif'), os.path.join(caiman_datadir(), 'split', 'second3000-ch1.tif')] is_patches = True # flag for processing in patches or not fr = 1.5 # approximate frame rate of data decay_time = 5.0 # length of transient if is_patches: # PROCESS IN PATCHES AND THEN COMBINE rf = 20 # half size of each patch stride = 4 # overlap between patches K = 2 # number of components in each patch else: # PROCESS THE WHOLE FOV AT ONCE rf = None # setting these parameters to None stride = None # will run CNMF on the whole FOV K = 10 # number of neurons expected (in the whole FOV) gSig = [6, 6] # expected half size of neurons merge_thresh = 0.80 # merging threshold, max correlation allowed p = 2 # order of the autoregressive system gnb = 2 # global background order params_dict = {'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'rf': rf, 'stride': stride, 'K': K, 'gSig': gSig, 'merge_thr': merge_thresh, 'p': p, 'nb': gnb} opts = params.CNMFParams(params_dict=params_dict) # %% Now RUN CaImAn Batch (CNMF) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) #cnm.estimates.normalize_components() cnm = cnm.fit_file() # %% plot contour plots of components Cn = cm.load(fnames[0], subindices=slice(1000)).local_correlations(swap_dim=False) cnm.estimates.plot_contours(img=Cn) # %% load memory mapped file Yr, dims, T = cm.load_memmap(cnm.mmap_file) images = np.reshape(Yr.T, [T] + list(dims), order='F') # %% refit cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier (this will pick up only neurons # and filter out active processes) min_SNR = 2 # peak SNR for accepted components (if above this, acept) rval_thr = 0.85 # space correlation threshold (if above this, accept) use_cnn = False # use the CNN classifier min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected cnm2.params.set('quality', {'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': use_cnn, 'min_cnn_thr': min_cnn_thr, 'cnn_lowest': cnn_lowest}) cnm2.estimates.detrend_df_f() cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview) # %% visualize selected and rejected components cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components) # %% visualize selected components cnm2.estimates.nb_view_components(images, idx=cnm2.estimates.idx_components, img=Cn) cnm2.estimates.view_components(images, idx=cnm2.estimates.idx_components_bad, img=Cn) #%% only select high quality components cnm2.estimates.select_components(use_object=True) #%% cnm2.estimates.plot_contours(img=Cn) cnm2.estimates.detrend_df_f() import pickle f = open("/home/david/zebraHorse/df_f_day55.pkl", "wb") pickle.dump(cnm2.estimates.F_dff, f) f.close() # %% play movie with results (original, reconstructed, amplified residual) for j in range(10): cnm2.estimates.play_movie(images, magnification=4.0, frame_range = range(100 * j, 100 * (j + 1))) #import time #time.sleep(1000) # %% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file)
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set to true to show movies and images fnames = ['data_endoscope.tif'] # filename to be processed fr = 10 # movie frame rate decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters do_motion_correction_nonrigid = False do_motion_correction_rigid = True # choose motion correction type gSig_filt = (3, 3) # size of filter, in general gSig (see below), # change this one if algorithm does not work max_shifts = (5, 5) # maximum allowed rigid shift strides = ( 48, 48 ) # start a new patch for pw-rigid motion correction every x pixels overlaps = (24, 24 ) # overlap between pathes (size of patch strides+overlaps) # for parallelization split the movies in num_splits chuncks across time # (make sure that length_movie/num_splits_to_process_rig>100) splits_rig = 10 splits_els = 10 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 #%% start the cluster try: cm.stop_server() # stop it if it was running except (): pass c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes= 24, # number of process to use, if you go out of memory try to reduce this one single_thread=False) #%% download demo file fnames = [download_demo(fnames[0])] filename_reorder = fnames #%% MOTION CORRECTION if do_motion_correction_nonrigid or do_motion_correction_rigid: # do motion correction rigid mc = motion_correct_oneP_rigid( fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, dview=dview, splits_rig=splits_rig, save_movie=not (do_motion_correction_nonrigid), border_nan='copy') new_templ = mc.total_template_rig plt.subplot(1, 2, 1) plt.imshow(new_templ) # % plot template plt.subplot(1, 2, 2) plt.plot(mc.shifts_rig) # % plot rigid shifts plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames') plt.ylabel('pixels') # borders to eliminate from movie because of motion correction bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) filename_reorder = mc.fname_tot_rig # do motion correction nonrigid if do_motion_correction_nonrigid: mc = motion_correct_oneP_nonrigid( fnames, gSig_filt=gSig_filt, max_shifts=max_shifts, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, dview=dview, splits_rig=None, save_movie=True, # whether to save movie in memory mapped format new_templ=new_templ, # template to initialize motion correction border_nan='copy') filename_reorder = mc.fname_tot_els bord_px = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # create memory mappable file in the right order on the hard drive (C order) fname_new = cm.save_memmap(filename_reorder, base_name='memmap_', order='C', border_to_0=bord_px, dview=dview) # load memory mappable file Yr, dims, T = cm.load_memmap(fname_new) Y = Yr.T.reshape((T, ) + dims, order='F') #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None gSig = ( 3, 3 ) # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1 Ain = None # possibility to seed with predetermined binary masks merge_thresh = .7 # merging threshold, max correlation allowed rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80 stride_cnmf = 20 # amount of overlap between the patches in pixels # (keep it at least large as gSiz, i.e 4 times the neuron size gSig) tsub = 2 # downsampling factor in time for initialization, # increase if you have memory problems ssub = 1 # downsampling factor in space for initialization, # increase if you have memory problems # you can pass them here as boolean vectors low_rank_background = None # None leaves background of each patch intact, # True performs global low-rank approximation if gnb>0 gnb = 0 # number of background components (rank) if positive, # else exact ring model with following settings # gnb= 0: Return background as b and W # gnb=-1: Return full rank background B # gnb<-1: Don't return background nb_patch = 0 # number of background components (rank) per patch if gnb>0, # else it is set automatically min_corr = .8 # min peak value from correlation image min_pnr = 10 # min peak to noise ration from PNR image ssub_B = 2 # additional downsampling factor in space for background ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor # parameters for component evaluation min_SNR = 3 # adaptive way to set threshold on the transient size r_values_min = 0.85 # threshold on space consistency (if you lower more components # will be accepted, potentially with worst quality) opts = params.CNMFParams( dims=dims, fr=fr, decay_time=decay_time, method_init='corr_pnr', # use this for 1 photon k=K, gSig=gSig, gSiz=gSiz, merge_thresh=merge_thresh, p=p, tsub=tsub, ssub=ssub, rf=rf, stride=stride_cnmf, only_init_patch=True, # set it to True to run CNMF-E gnb=gnb, nb_patch=nb_patch, method_deconvolution='oasis', # could use 'cvxpy' alternatively low_rank_background=low_rank_background, update_background_components= True, # sometimes setting to False improve the results min_corr=min_corr, min_pnr=min_pnr, normalize_init=False, # just leave as is center_psf=True, # leave as is for 1 photon ssub_B=ssub_B, ring_size_factor=ring_size_factor, del_duplicates=True, # whether to remove duplicates from initialization border_pix=bord_px) # number of pixels to not consider in the borders) #%% compute some summary images (correlation and peak to noise) # change swap dim if output looks weird, it is a problem with tiffile cn_filter, pnr = cm.summary_images.correlation_pnr(Y, gSig=gSig[0], swap_dim=False) # inspect the summary images and set the parameters inspect_correlation_pnr(cn_filter, pnr) # print parameters set above, modify them if necessary based on summary images print(min_corr) # min correlation of peak (from correlation image) print(min_pnr) # min peak to noise ratio #%% RUN CNMF ON PATCHES cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts) cnm.fit(Y) #%% DISCARD LOW QUALITY COMPONENTS cnm.params.set('quality', { 'min_SNR': min_SNR, 'rval_thr': r_values_min, 'use_cnn': False }) cnm.estimates.evaluate_components(Y, cnm.params, dview=dview) print(' ***** ') print('Number of total components: ', len(cnm.estimates.C)) print('Number of accepted components: ', len(cnm.estimates.idx_components)) #%% PLOT COMPONENTS cnm.dims = dims if display_images: cnm.estimates.plot_contours(img=cn_filter, idx=cnm.estimates.idx_components) cnm.estimates.view_components(Y, idx=cnm.estimates.idx_components) #%% MOVIES if display_images: # fully reconstructed movie cnm.estimates.play_movie(Y, q_max=99.9, magnification=2, include_bck=True, gain_res=10, bpx=bord_px) # movie without background cnm.estimates.play_movie(Y, q_max=99.9, magnification=2, include_bck=False, gain_res=4, bpx=bord_px) #%% STOP SERVER cm.stop_server(dview=dview)
border_nan = 'copy' mc_dict = { 'fnames': fnames, 'fr': fr, 'decay_time': decay_time, 'pw_rigid': pw_rigid, 'max_shifts': max_shifts, 'gSig_filt': gSig_filt, 'strides': strides, 'overlaps': overlaps, 'max_deviation_rigid': max_deviation_rigid, 'border_nan': border_nan } opts = params.CNMFParams(params_dict=mc_dict) if motion_correct: # do motion correction rigid mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion')) mc.motion_correct(save_movie=True) fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig if pw_rigid: bord_px = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) else: bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) plt.subplot(1, 2, 1); plt.imshow(mc.total_template_rig) # % plot template plt.subplot(1, 2, 2); plt.plot(mc.shifts_rig) # % plot rigid shifts plt.legend(['x shifts', 'y shifts']) plt.xlabel('frames')
## path to motion corrected tif file folder = '/projects/p30771/miniscope/data/GRIN011/1_24_2019/H10_M19_S59/TIFs/full_movie_caiman_analysis/' #mc_file = 'memmap__d1_480_d2_752_d3_1_order_C_frames_202_.mmap' memory_map_file = sys.argv[1] # In[4]: # load memory mappable file print('loading file:') print(memory_map_file) Yr, dims, T = cm.load_memmap(memory_map_file) images = Yr.T.reshape((T, ) + dims, order='F') # opts = params.CNMFParams(params_dict={}) # bord_px = 0 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system K = None # upper bound on number of components per patch, in general None gSig = ( 3, 3 ) # gaussian width of a 2D gaussian kernel, which approximates a neuron gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1 Ain = None # possibility to seed with predetermined binary masks merge_thresh = .7 # merging threshold, max correlation allowed rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80 stride_cnmf = 20 # amount of overlap between the patches in pixels # (keep it at least large as gSiz, i.e 4 times the neuron size gSig) tsub = 2 # downsampling factor in time for initialization,