def mc_vids(vids_fpath, mc_rigid_template): start = time.time() # estimated minimum value of the movie to produce an output that is positive min_mov = np.array([ cm.motion_correction.high_pass_filter_space(m_, gSig_filt) for m_ in cm.load(vids_fpath[0], subindices=range(400)) ]).min() mc = MotionCorrect(vids_fpath, min_mov, dview=dview, max_shifts=max_shifts, niter_rig=1, splits_rig=splits_rig, num_splits_to_process_rig=None, shifts_opencv=True, nonneg_movie=True, gSig_filt=gSig_filt, border_nan=border_nan, is3D=False) mc.motion_correct_rigid(save_movie=(not doPwRigid), template=mc_rigid_template) shifts_rig = mc.shifts_rig template_rig = mc.total_template_rig if doPwRigid: mc.motion_correct_pwrigid(save_movie=True, template=template_rig) mc.total_template_rig = template_rig duration = time.time() - start logging.info('Motion correction done in %s', str(duration)) return mc, duration, shifts_rig
def _run_motion_correction(self, file_name, max_shifts, strides, overlaps, upsample_factor_grid, max_deviation_rigid): """ Private function that initiates motion correction from CaImAn package, and return the motion corrected file names and their respective pixel shifts. """ offset_orig = np.nanmin(self.data_orig[:1000]) G6Image = MotionCorrect(file_name, offset_orig, max_shifts=max_shifts, niter_rig=1, splits_rig=56, strides=strides, overlaps=overlaps, splits_els=56, upsample_factor_grid=upsample_factor_grid, shifts_opencv=True, max_deviation_rigid=max_deviation_rigid, nonneg_movie=True) G6Image.motion_correct_rigid(save_movie=True) G6Image.motion_correct_pwrigid(save_movie=True, template=G6Image.total_template_rig) name_rig = G6Image.fname_tot_rig name_pwrig = G6Image.fname_tot_els shifts_rig = G6Image.shifts_rig x_shifts_pwrig = G6Image.x_shifts_els y_shifts_pwrig = G6Image.y_shifts_els template_shape = G6Image.total_template_els.shape return name_rig, name_pwrig, shifts_rig, x_shifts_pwrig, y_shifts_pwrig, template_shape
def motion_correct_pwrigid(self, fname): dview = None try: c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) niter_rig = 1 # number of iterations for rigid motion correction max_shifts = self.get_dict_param( 'max_shifts_pwrigid', 'tuple_int') # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = self.get_dict_param('splits_rig', 'single_int') # start a new patch for pw-rigid motion correction every x pixels strides = self.get_dict_param('strides', 'tuple_int') # overlap between pathes (size of patch strides+overlaps) overlaps = self.get_dict_param('overlaps', 'tuple_int') # for parallelization split the movies in num_splits chuncks across time splits_els = self.get_dict_param('splits_els', 'single_int') upsample_factor_grid = self.get_dict_param( 'upsample_factor_grid', 'single_int' ) # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = self.get_dict_param('max_deviation_rigid', 'single_int') # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative print( str([ max_shifts, splits_rig, strides, overlaps, splits_els, upsample_factor_grid, max_deviation_rigid, min_mov ])) mc = MotionCorrect(fname, min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, border_nan='copy', upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) mc.motion_correct_pwrigid(save_movie=True) self.motion_correct = mc except Exception as e: raise e finally: cm.cluster.stop_server(dview=dview)
def motion_correct(video_path, max_shift, patch_stride, patch_overlap, use_multiprocessing=True): full_video_path = video_path directory = os.path.dirname(full_video_path) filename = os.path.basename(full_video_path) memmap_video = tifffile.memmap(video_path) if use_multiprocessing: if os.name == 'nt': backend = 'multiprocessing' else: backend = 'ipyparallel' # Create the cluster cm.stop_server() c, dview, n_processes = cm.cluster.setup_cluster(backend=backend, n_processes=None, single_thread=False) else: dview = None z_range = list(range(memmap_video.shape[1])) new_video_path = os.path.join(directory, "mc_video_temp.tif") shutil.copyfile(video_path, new_video_path) mc_video = tifffile.memmap(new_video_path).astype(np.uint16) mc_borders = [ None for z in z_range ] counter = 0 for z in z_range: print("Motion correcting plane z={}...".format(z)) video_path = os.path.join(directory, os.path.splitext(filename)[0] + "_z_{}_temp.tif".format(z)) tifffile.imsave(video_path, memmap_video[:, z, :, :]) mc_video[:, z, :, :] *= 0 # --- PARAMETERS --- # params_movie = {'fname': video_path, 'max_shifts': (max_shift, max_shift), # maximum allow rigid shift (2,2) 'niter_rig': 3, 'splits_rig': 1, # for parallelization split the movies in num_splits chuncks across time 'num_splits_to_process_rig': None, # if none all the splits are processed and the movie is saved 'strides': (patch_stride, patch_stride), # intervals at which patches are laid out for motion correction 'overlaps': (patch_overlap, patch_overlap), # overlap between pathes (size of patch strides+overlaps) 'splits_els': 1, # for parallelization split the movies in num_splits chuncks across time 'num_splits_to_process_els': [None], # if none all the splits are processed and the movie is saved 'upsample_factor_grid': 4, # upsample factor to avoid smearing when merging patches 'max_deviation_rigid': 3, # maximum deviation allowed for patch with respect to rigid shift } # load movie (in memory!) fname = params_movie['fname'] niter_rig = params_movie['niter_rig'] # maximum allow rigid shift max_shifts = params_movie['max_shifts'] # for parallelization split the movies in num_splits chuncks across time splits_rig = params_movie['splits_rig'] # if none all the splits are processed and the movie is saved num_splits_to_process_rig = params_movie['num_splits_to_process_rig'] # intervals at which patches are laid out for motion correction strides = params_movie['strides'] # overlap between pathes (size of patch strides+overlaps) overlaps = params_movie['overlaps'] # for parallelization split the movies in num_splits chuncks across time splits_els = params_movie['splits_els'] # if none all the splits are processed and the movie is saved num_splits_to_process_els = params_movie['num_splits_to_process_els'] # upsample factor to avoid smearing when merging patches upsample_factor_grid = params_movie['upsample_factor_grid'] # maximum deviation allowed for patch with respect to rigid # shift max_deviation_rigid = params_movie['max_deviation_rigid'] # --- RIGID MOTION CORRECTION --- # # Load the original movie m_orig = tifffile.memmap(fname) # m_orig = cm.load(fname) min_mov = np.min(m_orig) # movie must be mostly positive for this to work offset_mov = -min_mov # Create motion correction object mc = MotionCorrect(fname, min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, num_splits_to_process_rig=num_splits_to_process_rig, strides= strides, overlaps= overlaps, splits_els=splits_els, num_splits_to_process_els=num_splits_to_process_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv = True, nonneg_movie = True, border_nan='min') # Do rigid motion correction mc.motion_correct_rigid(save_movie=False) # --- ELASTIC MOTION CORRECTION --- # # Do elastic motion correction mc.motion_correct_pwrigid(save_movie=True, template=mc.total_template_rig, show_template=False) # # Save elastic shift border bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # np.savez(mc.fname_tot_els + "_bord_px_els.npz", bord_px_els) fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_z_{}'.format(z), order = 'C', border_to_0 = bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') mc_borders[z] = bord_px_els # images += np.amin(images) # print(np.amax(images)) # print(np.amin(images)) # print(type(images)) mc_video[:, z, :, :] = (images - np.amin(images)).astype(np.uint16) del m_orig os.remove(video_path) try: os.remove(mc.fname_tot_rig) os.remove(mc.fname_tot_els) except: pass counter += 1 if use_multiprocessing: if backend == 'multiprocessing': dview.close() else: try: dview.terminate() except: dview.shutdown() cm.stop_server() mmap_files = glob.glob(os.path.join(directory, '*.mmap')) for mmap_file in mmap_files: try: os.remove(mmap_file) except: pass log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file) return mc_video, new_video_path, mc_borders
min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # maximum shift to be used for trimming against NaNs #%% compare with original movie cm.concatenate([ m_orig.resize(1, 1, downsample_ratio) + offset_mov, m_els.resize(1, 1, downsample_ratio) ], axis=2).play(fr=60, gain=15, magnification=2, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C'
dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides= strides, overlaps= overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv = True, nonneg_movie = True) # note that the file is not loaded in memory #%% mc.motion_correct_rigid(save_movie=True) m_els = cm.load(mc.fname_tot_rig[0]) bord_px_els = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) fname_cor = mc.fname_tot_rig #%% Run piecewise-rigid motion correction using NoRMCorre non_rigid = False if non_rigid: mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) fname_cor = mc.fname_tot_els # maximum shift to be used for trimming against NaNs #%% compare with original movie cm.concatenate([m_orig.resize(1, 1, downsample_ratio)+offset_mov, m_els.resize(1, 1, downsample_ratio)], axis=2).play(fr=60, gain=5, magnification=1, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' fnames = fname_cor # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order = 'C',
def run_motion_correction(cropping_file, dview): """ This is the function for motion correction. Its goal is to take in a decoded and cropped .tif file, perform motion correction, and save the result as a .mmap file. This function is only runnable on the cn76 server because it requires parallel processing. Args: cropping_file: tif file after cropping dview: cluster Returns: row: pd.DataFrame object The row corresponding to the motion corrected analysis state. """ # Get output file paths data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/motion_correction/' sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,input,home_path,decoding_main FROM Analysis WHERE cropping_main=? ORDER BY motion_correction_v" val = [ cropping_file, ] cursor.execute(sql, val) result = cursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[6] == 0: data[6] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}" output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl' sql1 = "UPDATE Analysis SET motion_correction_meta=?,motion_correction_v=? WHERE cropping_main=? " val1 = [output_meta_pkl_file_path, data[6], cropping_file] cursor.execute(sql1, val1) else: data[6] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}" output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl' sql2 = "INSERT INTO Analysis (motion_correction_meta,motion_correction_v) VALUES (?,?)" val2 = [output_meta_pkl_file_path, data[6]] cursor.execute(sql2, val2) database.commit() sql3 = "UPDATE Analysis SET decoding_main=?,decoding_v=?,mouse=?,session=?,trial=?,is_rest=?,input=?,home_path=?,cropping_v=?,cropping_main=? WHERE motion_correction_meta=? AND motion_correction_v=?" val3 = [ data[9], data[4], data[0], data[1], data[2], data[3], data[7], data[8], data[5], cropping_file, output_meta_pkl_file_path, data[6] ] cursor.execute(sql3, val3) database.commit() output_meta_pkl_file_path_full = data_dir + output_meta_pkl_file_path # Calculate movie minimum to subtract from movie cropping_file_full = os.environ['DATA_DIR_LOCAL'] + cropping_file min_mov = np.min(cm.load(cropping_file_full)) # Apply the parameters to the CaImAn algorithm sql5 = "SELECT motion_correct,pw_rigid,save_movie_rig,gSig_filt,max_shifts,niter_rig,strides,overlaps,upsample_factor_grid,num_frames_split,max_deviation_rigid,shifts_opencv,use_conda,nonneg_movie, border_nan FROM Analysis WHERE cropping_main=? " val5 = [ cropping_file, ] cursor.execute(sql5, val5) myresult = cursor.fetchall() para = [] aux = [] for x in myresult: aux = x for y in aux: para.append(y) parameters = { 'motion_correct': para[0], 'pw_rigid': para[1], 'save_movie_rig': para[2], 'gSig_filt': (para[3], para[3]), 'max_shifts': (para[4], para[4]), 'niter_rig': para[5], 'strides': (para[6], para[6]), 'overlaps': (para[7], para[7]), 'upsample_factor_grid': para[8], 'num_frames_split': para[9], 'max_deviation_rigid': para[10], 'shifts_opencv': para[11], 'use_cuda': para[12], 'nonneg_movie': para[13], 'border_nan': para[14] } caiman_parameters = parameters.copy() caiman_parameters['min_mov'] = min_mov opts = params.CNMFParams(params_dict=caiman_parameters) # Rigid motion correction (in both cases) logging.info('Performing rigid motion correction') t0 = datetime.datetime.today() # Create a MotionCorrect object mc = MotionCorrect([cropping_file_full], dview=dview, **opts.get_group('motion')) # Perform rigid motion correction mc.motion_correct_rigid(save_movie=parameters['save_movie_rig'], template=None) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes logging.info(f' Rigid motion correction finished. dt = {dt} min') # Obtain template, rigid shifts and border pixels total_template_rig = mc.total_template_rig shifts_rig = mc.shifts_rig # Save template, rigid shifts and border pixels in a dictionary meta_pkl_dict = { 'rigid': { 'template': total_template_rig, 'shifts': shifts_rig, } } sql = "UPDATE Analysis SET duration_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [dt, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) if parameters['save_movie_rig'] == 1: # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f' Loading rigid movie for cropping') m_rig = cm.load(mc.fname_tot_rig[0]) logging.info(f' Loaded rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_rigid_shifts(shifts_rig) # Crop the movie logging.info( f' Cropping and saving rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_rig = m_rig.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['rigid']['cropping_points'] = [x_, _x, y_, _y] sql = "UPDATE Analysis SET motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [x_, _x, y_, _y, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) # Save the movie rig_role = 'alternate' if parameters['pw_rigid'] else 'main' fname_tot_rig = m_rig.save(data_dir + rig_role + '/' + file_name + '_rig' + '.mmap', order='C') logging.info(f' Cropped and saved rigid movie as {fname_tot_rig}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) sql = "UPDATE Analysis SET motion_correction_rig_role=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [fname_tot_rig, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) database.commit() # If specified in the parameters, apply piecewise-rigid motion correction if parameters['pw_rigid'] == 1: logging.info(f' Performing piecewise-rigid motion correction') t0 = datetime.datetime.today() # Perform non-rigid (piecewise rigid) motion correction. Use the rigid result as a template. mc.motion_correct_pwrigid(save_movie=True, template=total_template_rig) # Obtain template and filename total_template_els = mc.total_template_els fname_tot_els = mc.fname_tot_els[0] dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes meta_pkl_dict['pw_rigid'] = { 'template': total_template_els, 'x_shifts': mc.x_shifts_els, 'y_shifts': mc. y_shifts_els # removed them initially because they take up space probably } logging.info( f' Piecewise-rigid motion correction finished. dt = {dt} min') # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f' Loading pw-rigid movie for cropping') m_els = cm.load(fname_tot_els) logging.info(f' Loaded pw-rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_pw_rigid_shifts( np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) # Crop the movie logging.info( f' Cropping and saving pw-rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_els = m_els.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] # Save the movie fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') logging.info(f'Cropped and saved rigid movie as {fname_tot_els}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_els[0]) sql = "UPDATE Analysis SET motion_correction_main=?, motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=?,duration_pw_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [ fname_tot_els, x_, _x, y_, _y, dt, output_meta_pkl_file_path, data[6] ] cursor.execute(sql, val) database.commit() # Write meta results dictionary to the pkl file pkl_file = open(output_meta_pkl_file_path_full, 'wb') pickle.dump(meta_pkl_dict, pkl_file) pkl_file.close() return fname_tot_els, data[6]
def run_single(batch_dir, UUID, output): file_path = os.path.join(batch_dir, UUID) n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False, ignore_preexisting=True) fname = [file_path + '_input.tiff'] # TODO: Should just unpack the input params as kwargs input_params: dict = pickle.load(open(file_path + '.params', 'rb')) mc_kwargs = input_params['mc_kwargs'] splits_rig = n_processes splits_els = n_processes if os.environ['_MESMERIZE_USE_CUDA'] == 'True': USE_CUDA = True else: USE_CUDA = False min_mov = cm.load(fname[0], subindices=range(200)).min() mc = MotionCorrect(fname[0], min_mov, dview=dview, splits_rig=splits_rig, splits_els=splits_els, shifts_opencv=True, nonneg_movie=True, use_cuda=USE_CUDA, **mc_kwargs) if 'template' in input_params.keys(): template = input_params['template'] else: template = None mc.motion_correct_pwrigid(save_movie=True, template=template) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) m_els -= np.nanmin(m_els) if input_params['output_bit_depth'] == 'Do not convert': pass elif input_params['output_bit_depth'] == '8': m_els = m_els.astype(np.uint8, copy=False) elif input_params['output_bit_depth'] == '16': m_els = m_els.astype(np.uint16) img_out_path = os.path.join(batch_dir, f'{UUID}_mc.tiff') tifffile.imsave(img_out_path, m_els, bigtiff=True, imagej=False, compress=1) output['output_files'] = [UUID + '_mc.tiff'] output.update({'status': 1, 'bord_px': int(bord_px_els)}) if not input_params.get('keep_memmap', False): for mf in glob(os.path.join(batch_dir, UUID + '*.mmap')): try: os.remove(mf) except: pass dview.terminate() return output
def run_multi(batch_dir, UUID, output): file_path = os.path.join(batch_dir, UUID) n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) filename = [file_path + '_input.tiff'] seq = tifffile.TiffFile(filename[0]).asarray() seq_shape = seq.shape # assume default tzxy for z in range(seq.shape[1]): tifffile.imsave(f'{file_path}_z{z}.tiff', seq[:, z, :, :]) del seq print("******** Creating process pool ********") c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False, ignore_preexisting=True) # TODO: Should just unpack the input params as kwargs input_params = pickle.load(open(file_path + '.params', 'rb')) mc_kwargs = input_params['mc_kwargs'] splits_rig = n_processes splits_els = n_processes if os.environ['_MESMERIZE_USE_CUDA'] == 'True': USE_CUDA = True else: USE_CUDA = False output_files = [] for z in range(seq_shape[1]): print(f"Plane {z} / {seq_shape[1]}") filename = [f'{file_path}_z{z}.tiff'] print('Creating memmap') min_mov = cm.load(filename[0], subindices=range(200)).min() mc = MotionCorrect(filename[0], min_mov, dview=dview, splits_rig=splits_rig, splits_els=splits_els, shifts_opencv=True, nonneg_movie=True, use_cuda=USE_CUDA, **mc_kwargs) mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) m_els -= np.nanmin(m_els) if input_params['output_bit_depth'] == 'Do not convert': pass elif input_params['output_bit_depth'] == '8': m_els = m_els.astype(np.uint8, copy=False) elif input_params['output_bit_depth'] == '16': m_els = m_els.astype(np.uint16) if z == 0: mc_out = np.zeros(seq_shape, dtype=m_els.dtype) mc_out[:, z, :, :] = m_els img_out_path = os.path.join(batch_dir, f'{UUID}_mc.tiff') tifffile.imsave(img_out_path, mc_out, bigtiff=True, imagej=False, compress=1) output['output_files'] = [f'{UUID}_mc.tiff'] output.update({ 'status': 1, 'bord_px': int(bord_px_els), }) for mf in glob(os.path.join(batch_dir, UUID + '*.mmap')): try: os.remove(mf) except: pass dview.terminate() return output
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # num processes n_proc = 12 # dataset dependent parameters fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_proc, single_thread=False) print('Parallel processing initialized.') print('Beginning motion correction') #%%% MOTION CORRECTION t_ms = time.time() # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative if not nomc: mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig,splits_rig=splits_rig,strides=strides, overlaps=overlaps, splits_els=splits_els,upsample_factor_grid=upsample_factor_grid,max_deviation_rigid=max_deviation_rigid,shifts_opencv=True, nonneg_movie=True, use_cuda=use_cuda) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),np.max(np.abs(mc.y_shifts_els)))).astype(np.int) t_mf = time.time() print('Motion correction complete in ', int(t_mf - t_ms),' seconds') # maximum shift to be used for trimming against NaNs #%% compare with original movie #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders bord_px_els = 5 # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_proc, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) print('Beginning initial CNMF fit') t1 = time.time() cnm = cnmf.CNMF(n_processes=n_proc, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) t2 = time.time() print('Initial CNMF fit complete in ', int(t2-t1), 'seconds.') Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier t3 = time.time() print('Estimating component quality') idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) t4 = time.time() print('Component quality estimation complete in ', int(t4-t3),' seconds') #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution print('Re-running CNMF to refine and deconvolve') t5 = time.time() A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=n_proc, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) t6 = time.time() print('CNMF re-run complete in ', int(t6-t5), ' seconds') #%% Extract DF/F values # F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, # quantileMin=8, frames_window=250) save_results = True if save_results: if os.path.isdir(infile[0]): outfile = os.path.join(infile[0],'caiman_output.npz') else: outfile = os.path.join(os.path.split(infile[0])[0],'caiman_output.npz') np.savez_compressed(outfile,Cn=Cn, A=cnm2.A.todense(), C=cnm2.C,b=cnm2.b, f=cnm2.f, YrA=cnm2.YrA, d1=d1, d2=d2,idx_components=idx_components, idx_components_bad=idx_components_bad) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview)
def run(batch_dir: str, UUID: str): start_time = time() output = {'status': 0, 'output_info': ''} file_path = batch_dir + '/' + UUID n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', # use this one n_processes=n_processes, # number of process to use, if you go out of memory try to reduce this one single_thread=False) try: fname = [file_path + '.tiff'] input_params = pickle.load(open(file_path + '.params', 'rb')) # TODO: Should just unpack the input params as kwargs niter_rig = input_params['iters_rigid'] max_shifts = (input_params['max_shifts_x'], input_params['max_shifts_y']) splits_rig = n_processes strides = (input_params['strides'], input_params['strides']) overlaps = (input_params['overlaps'], input_params['overlaps']) splits_els = n_processes upsample_factor_grid = input_params['upsample'] max_deviation_rigid = input_params['max_dev'] if 'gSig_filt' in input_params.keys(): gSig_filt = input_params['gSig_filt'] else: gSig_filt = None if os.environ['_MESMERIZE_USE_CUDA'] == 'True': USE_CUDA = True else: USE_CUDA = False min_mov = cm.load(fname[0], subindices=range(200)).min() mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True, use_cuda=USE_CUDA, gSig_filt=gSig_filt) mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # p = pickle.load(open(UUID + '_workEnv.pik', 'rb')) # if p['imdata']['meta']['origin'] == 'mes': # if p['imdata']['meta']['orig_meta']['DataType'] == 'uint16': # pass # # lut = BitDepthConverter.create_lut([np.nanmin(m_els), np.nanmax(m_els)], source=16, out=8) # # else: m_els -= np.nanmin(m_els) if input_params['output_bit_depth'] == 'Do not convert': pass elif input_params['output_bit_depth'] == '8': m_els = m_els.astype(np.uint8, copy=False) elif input_params['output_bit_depth'] == '16': m_els = m_els.astype(np.uint16) tifffile.imsave(batch_dir + '/' + UUID + '_mc.tiff', m_els, bigtiff=True, imagej=True, compress=1) output.update({'status': 1, 'bord_px': int(bord_px_els)}) except Exception: output.update({'status': 0, 'output_info': traceback.format_exc()}) for mf in glob(batch_dir + '/' + UUID + '*.mmap'): os.remove(mf) dview.terminate() end_time = time() processing_time = (end_time - start_time) / 60 output_files_list = [UUID + '_mc.tiff', UUID + '.out'] output.update({ 'processing_time': processing_time, 'output_files': output_files_list }) json.dump(output, open(file_path + '.out', 'w'))
def caiman_main_light_weight(fr, fnames, z=0, dend=False): """ Main function to compute the caiman algorithm. For more details see github and papers fpath(str): Folder where to store the plots fr(int): framerate fnames(list-str): list with the names of the files to be computed together z(array): vector with the values of z relative to y dend(bool): Boleean to change parameters to look for neurons or dendrites display_images(bool): to display and save different plots returns F_dff(array): array with the dff of the components com(array): matrix with the position values of the components as given by caiman cnm(struct): struct with different stimates and returns from caiman""" # parameters decay_time = 0.4 # length of a typical transient in seconds # Look for the best parameters for this 2p system and never change them again :) # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (3, 3) # maximum allow rigid shift splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time strides = ( 96, 96 ) # start a new patch for pw-rigid motion correction every x pixels overlaps = (48, 48 ) # overlap between pathes (size of patch strides+overlaps) splits_els = 10 # for parallelization split the movies in num_splits chuncks across time upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches max_deviation_rigid = 3 # maximum deviation allowed for patch with respect to rigid shifts # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed rf = 25 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 stride_cnmf = 10 # amount of overlap between the patches in pixels K = 25 # number of components per patch if dend: gSig = [1, 1] # expected half size of neurons init_method = 'sparse_nmf' # initialization method (if analyzing dendritic data using 'sparse_nmf') alpha_snmf = 1e-6 # sparsity penalty for dendritic data analysis through sparse NMF else: gSig = [3, 3] # expected half size of neurons init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf') alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier dview = None # parallel processing keeps crashing. print('***************Starting motion correction*************') print('files:') print(fnames) # %% start a cluster for parallel processing # c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fnames[0]).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fnames, min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory # %% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) totdes = [np.nansum(mc.x_shifts_els), np.nansum(mc.y_shifts_els)] print('***************Motion correction has ended*************') # maximum shift to be used for trimming against NaNs # %% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) # %% restart cluster to clean up memory # cm.stop_server(dview=dview) # c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) # %% RUN CNMF ON PATCHES print('***************Running CNMF...*************') # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) # %% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.estimates.A[:, idx_components], cnm.estimates.C[ idx_components], cnm.estimates.b, cnm.estimates.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) print('***************Fit*************') cnm2 = cnm2.fit(images) print('***************Extractind DFFs*************') # %% Extract DF/F values # cm.stop_server(dview=dview) try: F_dff = detrend_df_f(cnm2.estimates.A, cnm2.estimates.b, cnm2.estimates.C, cnm2.estimates.f, YrA=cnm2.estimates.YrA, quantileMin=8, frames_window=250) # F_dff = detrend_df_f(cnm.A, cnm.b, cnm.C, cnm.f, YrA=cnm.YrA, quantileMin=8, frames_window=250) except: F_dff = cnm2.estimates.C * np.nan print('WHAAT went wrong again?') print('***************stopping cluster*************') # %% STOP CLUSTER and clean up log files # cm.stop_server(dview=dview) # *************************************************************************************** # Preparing output data # F_dff -> DFF values, is a matrix [number of neurons, length recording] # com --> center of mass, is a matrix [number of neurons, 2] print('***************preparing output data*************') if len(dims) <= 2: if len(z) == 1: com = np.concatenate( (cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]), np.zeros((cnm2.estimates.A.shape[1], 1)) + z), 1) elif len(z) == dims[0]: auxcom = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]) zy = np.zeros((auxcom.shape[0], 1)) for y in np.arange(auxcom.shape[0]): zy[y, 0] = z[int(auxcom[y, 0])] com = np.concatenate((auxcom, zy), 1) else: print( 'WARNING: Z value was not correctly defined, only X and Y values on file, z==zeros' ) print(['length of z was: ' + str(len(z))]) com = np.concatenate( (cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]), np.zeros((cnm2.estimates.A.shape[1], 1))), 1) else: com = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1], dims[2]) return F_dff, com, cnm2, totdes, SNR_comp[idx_components]
def run(batch_dir: str, UUID: str): logging.basicConfig( stream=sys.stdout, level=logging.DEBUG, format= "%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s" ) start_time = time() output = {'status': 0, 'output_info': ''} file_path = os.path.join(batch_dir, UUID) n_processes = os.environ['_MESMERIZE_N_THREADS'] n_processes = int(n_processes) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False, ignore_preexisting=True) try: fname = [file_path + '_input.tiff'] input_params = pickle.load(open(file_path + '.params', 'rb')) # TODO: Should just unpack the input params as kwargs mc_kwargs = input_params['mc_kwargs'] splits_rig = n_processes splits_els = n_processes if os.environ['_MESMERIZE_USE_CUDA'] == 'True': USE_CUDA = True else: USE_CUDA = False min_mov = cm.load(fname[0], subindices=range(200)).min() mc = MotionCorrect(fname[0], min_mov, dview=dview, splits_rig=splits_rig, splits_els=splits_els, shifts_opencv=True, nonneg_movie=True, use_cuda=USE_CUDA, **mc_kwargs) mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) m_els -= np.nanmin(m_els) if input_params['output_bit_depth'] == 'Do not convert': pass elif input_params['output_bit_depth'] == '8': m_els = m_els.astype(np.uint8, copy=False) elif input_params['output_bit_depth'] == '16': m_els = m_els.astype(np.uint16) img_out_path = os.path.join(batch_dir, f'{UUID}_mc.tiff') tifffile.imsave(img_out_path, m_els, bigtiff=True, imagej=True, compress=1) output.update({'status': 1, 'bord_px': int(bord_px_els)}) except Exception: output.update({'status': 0, 'output_info': traceback.format_exc()}) for mf in glob(os.path.join(batch_dir, UUID + '*.mmap')): try: os.remove(mf) except: pass dview.terminate() end_time = time() processing_time = (end_time - start_time) / 60 output_files_list = [UUID + '_mc.tiff'] output.update({ 'processing_time': processing_time, 'output_files': output_files_list }) json.dump(output, open(file_path + '.out', 'w'))
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters for data and motion correction # dataset dependent parameters fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds dxy = (2., 2.) # spatial resolution in x and y in (um per pixel) max_shift_um = (12., 12.) # maximum shift in um patch_motion_um = (100., 100.) # patch size for non-rigid motion correction in um # motion correction parameters pwrigid_motion_correct = True # flag to select rigid vs pw_rigid motion correction max_shifts = tuple([int(a/b) for a, b in zip(max_shift_um, dxy)]) # maximum allow rigid shift in pixels # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)]) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. # To close the video press q display_images = False if display_images: m_orig = cm.load_movie_chain(fname) downsample_ratio = 0.2 moviehandle = m_orig.resize(1, 1, downsample_ratio) moviehandle.play(q_max=99.5, fr=60, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname, min_mov, dview=dview, max_shifts=max_shifts, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, border_nan='copy', upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre if pwrigid_motion_correct: mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. else: mc.motion_correct_rigid(save_movie=True) m_els = cm.load(mc.fname_tot_rig) bord_px_els = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) fnames = mc.fname_tot_rig # name of the rigidly corrected file. # maximum shift to be used for trimming against NaNs #%% compare with original movie if display_images: downsample_ratio = 0.2 moviehandle = cm.concatenate([m_orig.resize(1, 1, downsample_ratio) - min_mov, m_els.resize(1, 1, downsample_ratio)], axis=2) moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' border_to_0 = bord_px_els # exclude borders due to motion correction # border_to_0 = 0 if mc.border_nan is 'copy' else bord_px_els # you can include boundaries if you used the 'copy' option in the motion # correction, although be careful abou the components near the boundaries fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=border_to_0) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=None, single_thread=False) #%% parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') method_init = 'greedy_roi' # parameters for component evaluation opts = params.CNMFParams(dims=dims, fr=fr, decay_time=decay_time, method_init=method_init, gSig=gSig, merge_thresh=merge_thresh, p=p, gnb=gnb, k=K, rf=rf, stride=stride_cnmf, rolling_sum=True) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) opts.set('temporal', {'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 cnm.estimates.plot_contours(img=Cn) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier cnm.params.set('quality', {'fr': fr, 'decay_time': decay_time, 'min_SNR': min_SNR, 'rval_thr': rval_thr, 'use_cnn': True, 'min_cnn_thr': cnn_thr}) cnm.estimates.evaluate_components(images, cnm.params, dview=dview) #%% PLOT COMPONENTS cnm.estimates.plot_contours(img=Cn, idx=cnm.estimates.idx_components) #%% VIEW TRACES (accepted and rejected) if display_images: cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components) cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components_bad) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution cnm.dview = None cnm2 = deepcopy(cnm) cnm2.dview = dview cnm2.params.set('patch', {'rf': None}) cnm2.params.set('temporal', {'p': p}) cnm2 = cnm2.fit(images) #%% Extract DF/F values cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250) #%% Show final traces cnm2.estimates.view_components(Yr, img=Cn) #%% reconstruct denoised movie (press q to exit) if display_images: cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2, magnification=2, bpx=border_to_0, include_bck=True) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file)
def main(): pass # For compatibility between running under Spyder and the CLI #%% First setup some parameters # dataset dependent parameters display_images = False # Set this to true to show movies and plots fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed fr = 30 # imaging rate in frames per second decay_time = 0.4 # length of a typical transient in seconds # motion correction parameters niter_rig = 1 # number of iterations for rigid motion correction max_shifts = (6, 6) # maximum allow rigid shift # for parallelization split the movies in num_splits chuncks across time splits_rig = 56 # start a new patch for pw-rigid motion correction every x pixels strides = (48, 48) # overlap between pathes (size of patch strides+overlaps) overlaps = (24, 24) # for parallelization split the movies in num_splits chuncks across time splits_els = 56 upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches # maximum deviation allowed for patch with respect to rigid shifts max_deviation_rigid = 3 # parameters for source extraction and deconvolution p = 1 # order of the autoregressive system gnb = 2 # number of global background components merge_thresh = 0.8 # merging threshold, max correlation allowed # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50 rf = 15 stride_cnmf = 6 # amount of overlap between the patches in pixels K = 4 # number of components per patch gSig = [4, 4] # expected half size of neurons # initialization method (if analyzing dendritic data using 'sparse_nmf') init_method = 'greedy_roi' is_dendrites = False # flag for analyzing dendritic data # sparsity penalty for dendritic data analysis through sparse NMF alpha_snmf = None # parameters for component evaluation min_SNR = 2.5 # signal to noise ratio for accepting a component rval_thr = 0.8 # space correlation threshold for accepting a component cnn_thr = 0.8 # threshold for CNN based classifier #%% download the dataset if it's not present in your folder if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']: fname = [download_demo(fname[0])] #%% play the movie # playing the movie using opencv. It requires loading the movie in memory. To # close the video press q m_orig = cm.load_movie_chain(fname[:1]) downsample_ratio = 0.2 offset_mov = -np.min(m_orig[:100]) moviehandle = m_orig.resize(1, 1, downsample_ratio) if display_images: moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2) #%% start a cluster for parallel processing c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%%% MOTION CORRECTION # first we create a motion correction object with the parameters specified min_mov = cm.load(fname[0], subindices=range(200)).min() # this will be subtracted from the movie to make it non-negative mc = MotionCorrect(fname[0], min_mov, dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig, strides=strides, overlaps=overlaps, splits_els=splits_els, upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid, shifts_opencv=True, nonneg_movie=True) # note that the file is not loaded in memory #%% Run piecewise-rigid motion correction using NoRMCorre mc.motion_correct_pwrigid(save_movie=True) m_els = cm.load(mc.fname_tot_els) bord_px_els = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # maximum shift to be used for trimming against NaNs #%% compare with original movie moviehandle = cm.concatenate([ m_orig.resize(1, 1, downsample_ratio) + offset_mov, m_els.resize(1, 1, downsample_ratio) ], axis=2) display_images = False if display_images: moviehandle.play(fr=60, q_max=99.5, magnification=2, offset=0) # press q to exit #%% MEMORY MAPPING # memory map the file in order 'C' fnames = mc.fname_tot_els # name of the pw-rigidly corrected file. border_to_0 = bord_px_els # number of pixels to exclude fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C', border_to_0=bord_px_els) # exclude borders # now load the file Yr, dims, T = cm.load_memmap(fname_new) d1, d2 = dims images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y) #%% restart cluster to clean up memory cm.stop_server(dview=dview) c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False) #%% RUN CNMF ON PATCHES # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) t1 = time.time() cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1, method_init=init_method, alpha_snmf=alpha_snmf, only_init_patch=False, gnb=gnb, border_pix=bord_px_els) cnm = cnm.fit(images) #%% plot contours of found components Cn = cm.local_correlations(images.transpose(1, 2, 0)) Cn[np.isnan(Cn)] = 0 plt.figure() crd = plot_contours(cnm.A, Cn, thr=0.9) plt.title('Contour plots of found components') #%% COMPONENT EVALUATION # the components are evaluated in three ways: # a) the shape of each component must be correlated with the data # b) a minimum peak SNR is required over the length of a transient # c) each shape passes a CNN based classifier idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \ estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, fr, decay_time, gSig, dims, dview=dview, min_SNR=min_SNR, r_values_min=rval_thr, use_cnn=False, thresh_cnn_min=cnn_thr) #%% PLOT COMPONENTS if display_images: plt.figure() plt.subplot(121) crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of accepted components') plt.subplot(122) crd_bad = cm.utils.visualization.plot_contours( cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75) plt.title('Contour plots of rejected components') #%% VIEW TRACES (accepted and rejected) if display_images: view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components], img=Cn) view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components_bad], cnm.C[idx_components_bad], cnm.b, cnm.f, dims[0], dims[1], YrA=cnm.YrA[idx_components_bad], img=Cn) #%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[ idx_components], cnm.b, cnm.f cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview, merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in, f_in=f_in, rf=None, stride=None, gnb=gnb, method_deconvolution='oasis', check_nan=True) cnm2 = cnm2.fit(images) #%% Extract DF/F values F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA, quantileMin=8, frames_window=250) #%% Show final traces cnm2.view_patches(Yr, dims=dims, img=Cn) #%% STOP CLUSTER and clean up log files cm.stop_server(dview=dview) log_files = glob.glob('*_LOG_*') for log_file in log_files: os.remove(log_file) #%% reconstruct denoised movie denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape( dims + (-1, ), order='F').transpose([2, 0, 1]) #%% play along side original data moviehandle = cm.concatenate([ m_els.resize(1, 1, downsample_ratio), denoised.resize(1, 1, downsample_ratio) ], axis=2) if display_images: moviehandle.play(fr=60, gain=15, magnification=2, offset=0) # press q to exit
# %% plot rigid shifts pl.close() pl.plot(mc.shifts_rig) pl.legend(['x shifts', 'y shifts']) pl.xlabel('frames') pl.ylabel('pixels') # %% inspect movie downsample_ratio = params_display['downsample_ratio'] # TODO: todocument offset_mov = -np.min(m_orig[:100]) m_rig.resize(1, 1, downsample_ratio).play( gain=10, offset=offset_mov * .25, fr=30, magnification=2, bord_px=bord_px_rig) # %% # a computing intensive but parralellized part t1 = time.time() mc.motion_correct_pwrigid(save_movie=True, template=mc.total_template_rig, show_template=True) # TODO: change var name els= pwr m_els = cm.load(mc.fname_tot_els) pl.imshow(mc.total_template_els, cmap='gray') # TODO: show screenshot 5 # TODO: bug sometimes saying there is no y_shifts_els bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype(np.int) # %% visualize elastic shifts pl.close() pl.subplot(2, 1, 1) pl.plot(mc.x_shifts_els) pl.ylabel('x shifts (pixels)') pl.subplot(2, 1, 2) pl.plot(mc.y_shifts_els) pl.ylabel('y_shifts (pixels)')
def main(index, row, parameters, dview): ''' This is the function for motion correction. Its goal is to take in a decoded and cropped .tif file, perform motion correction, and save the result as a .mmap file. This function is only runnable on the cn76 server because it requires parralel processing. Args: index: tuple The index of the analysis state to be motion corrected. row: pd.DataFrame object The row corresponding to the analysis state to be motion corrected. Returns: index: tuple The index of the motion corrected analysis state. row: pd.DataFrame object The row corresponding to the motion corrected analysis state. ''' # Forcing parameters if not parameters['pw_rigid']: parameters['save_movie_rig'] = True # Get input file input_tif_file_path = eval(row.loc['cropping_output'])['main'] if not os.path.isfile(input_tif_file_path): input_tif_file_path = src.pipeline.get_expected_file_path( 'cropping', index, 'main/', 'tif') if not os.path.isfile(input_tif_file_path): logging.error( 'Cropping file not found. Cancelling motion correction.') return index, row # Get output file paths data_dir = 'data/interim/motion_correction/' file_name = src.pipeline.create_file_name(step_index, index) output_meta_pkl_file_path = data_dir + f'meta/metrics/{file_name}.pkl' # Create a dictionary with the output output = { 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S") }, 'metrics': { 'other': output_meta_pkl_file_path } } } row.loc['motion_correction_parameters'] = str(parameters) # Calculate movie minimum to subtract from movie min_mov = np.min(cm.load(input_tif_file_path)) # Apply the parameters to the CaImAn algorithm caiman_parameters = parameters.copy() caiman_parameters['min_mov'] = min_mov opts = params.CNMFParams(params_dict=caiman_parameters) # Rigid motion correction (in both cases) logging.info(f'{index} Performing rigid motion correction') t0 = datetime.datetime.today() # Create a MotionCorrect object mc = MotionCorrect([input_tif_file_path], dview=dview, **opts.get_group('motion')) # Perform rigid motion correction mc.motion_correct_rigid(save_movie=parameters['save_movie_rig'], template=None) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes logging.info(f'{index} Rigid motion correction finished. dt = {dt} min') # Obtain template, rigid shifts and border pixels total_template_rig = mc.total_template_rig shifts_rig = mc.shifts_rig # Save template, rigid shifts and border pixels in a dictionary meta_pkl_dict = { 'rigid': { 'template': total_template_rig, 'shifts': shifts_rig, } } output['meta']['duration'] = {'rigid': dt} if parameters['save_movie_rig']: # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f'{index} Loading rigid movie for cropping') m_rig = cm.load(mc.fname_tot_rig[0]) logging.info(f'{index} Loaded rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_rigid_shifts(shifts_rig) # Crop the movie logging.info( f'{index} Cropping and saving rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_rig = m_rig.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['rigid']['cropping_points'] = [x_, _x, y_, _y] # Save the movie rig_role = 'alternate' if parameters['pw_rigid'] else 'main' fname_tot_rig = m_rig.save(data_dir + rig_role + '/' + file_name + '_rig' + '.mmap', order='C') logging.info( f'{index} Cropped and saved rigid movie as {fname_tot_rig}') # Store the total path in output output[rig_role] = fname_tot_rig # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) # If specified in the parameters, apply piecewise-rigid motion correction if parameters['pw_rigid']: logging.info(f'{index} Performing piecewise-rigid motion correction') t0 = datetime.datetime.today() # Perform non-rigid (piecewise rigid) motion correction. Use the rigid result as a template. mc.motion_correct_pwrigid(save_movie=True, template=total_template_rig) # Obtain template and filename total_template_els = mc.total_template_els fname_tot_els = mc.fname_tot_els[0] dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes meta_pkl_dict['pw_rigid'] = { 'template': total_template_els, 'x_shifts': mc.x_shifts_els, 'y_shifts': mc. y_shifts_els # removed them initially because they take up space probably } output['meta']['duration']['pw_rigid'] = dt logging.info( f'{index} Piecewise-rigid motion correction finished. dt = {dt} min' ) # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f'{index} Loading pw-rigid movie for cropping') m_els = cm.load(fname_tot_els) logging.info(f'{index} Loaded pw-rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_pw_rigid_shifts( np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) # Crop the movie logging.info( f'{index} Cropping and saving pw-rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_els = m_els.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] # Save the movie fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') logging.info( f'{index} Cropped and saved rigid movie as {fname_tot_els}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_els[0]) # Store the total path in output output['main'] = fname_tot_els # Write meta results dictionary to the pkl file pkl_file = open(output_meta_pkl_file_path, 'wb') pickle.dump(meta_pkl_dict, pkl_file) pkl_file.close() # Write necessary variables to the trial index and row row.loc['motion_correction_output'] = str(output) row.loc['motion_correction_parameters'] = str(parameters) # Compute the basic metrics 'crispness' get_metrics(index, row, crispness=True) # Create source extraction images in advance: logging.info(f'{index} Creating corr and pnr images in advance') index, row = src.steps.source_extraction.get_corr_pnr(index, row) logging.info(f'{index} Created corr and pnr images') return index, row