def run_cropper(input_path, parameters): """ This function takes in a decoded analysis state and crops it according to specified cropping points. Args: input_path the path of the decoding file """ # Determine output .tif file path sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,input,home_path FROM Analysis WHERE decoding_main=?" val = [input_path, ] mycursor.execute(sql, val) myresult = mycursor.fetchall() data = [] aux = [] for x in myresult: aux = x for y in aux: data.append(y) # update the database if data[5] == 0: data[5] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}" output_tif_file_path = f"data/interim/cropping/main/{file_name}.tif" sql1 = "UPDATE Analysis SET cropping_main=?,cropping_v=? WHERE decoding_main=? " val1 = [output_tif_file_path, data[5], input_path] mycursor.execute(sql1, val1) else: data[5] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}" output_tif_file_path = f"data/interim/cropping/main/{file_name}.tif" sql2 = "INSERT INTO Analysis (cropping_main,cropping_v) VALUES (?,?)" val2 = [output_tif_file_path, data[5]] mycursor.execute(sql2, val2) database.commit() sql3 = "UPDATE Analysis SET decoding_main=?,decoding_v=?,mouse=?,session=?,trial=?,is_rest=?,input=?,home_path=? WHERE cropping_main=? AND cropping_v=?" val3 = [input_path, data[4], data[0], data[1], data[2], data[3], data[6], data[7], output_tif_file_path, data[5]] mycursor.execute(sql3, val3) # Spatial cropping input_path = os.path.join(os.environ['DATA_DIR_LOCAL'], input_path) logging.info('Loading movie') m = cm.load(input_path) logging.info('Loaded movie') [x_, _x, y_, _y] = parameters['cropping_points_spatial'] logging.info('Performing spatial cropping') m = m[:, x_:_x, y_:_y] logging.info(' Spatial cropping finished') output_tif_file_path_full = os.path.join(os.environ['DATA_DIR_LOCAL'], output_tif_file_path) # Save the movie m.save(output_tif_file_path_full) return output_tif_file_path, data[5]
def cropping_interval(mouse): """ This function ask the user for cropping parameters """ # Choose crop parameters x1 = int(input("Limit X1 : ")) x2 = int(input("Limit X2 : ")) y1 = int(input("Limit Y1 : ")) y2 = int(input("Limit Y2 : ")) sql = "UPDATE Analysis SET crop_spatial=?,cropping_points_spatial_x1=?,cropping_points_spatial_x2=?,cropping_points_spatial_y1=?,cropping_points_spatial_y2=?,crop_temporal=?,cropping_points_temporal=? WHERE mouse = ?" val = [True, x1, x2, y1, y2, False, None, mouse] mycursor.execute(sql, val) database.commit() parameters_cropping = {'crop_spatial': True, 'cropping_points_spatial': [y1, y2, x1, x2], 'crop_temporal': False, 'cropping_points_temporal': [], 'segmentation': True} return parameters_cropping
def run_component_evaluation(input_file, session_wise=False, equalization=False): sql = "SELECT source_extraction_session_wise,min_SNR,alignment_main,equalization_main,motion_correction_main,rval_thr,use_cnn FROM Analysis WHERE source_extraction_main=?" val = [ input_file, ] mycursor.execute(sql, val) myresult = mycursor.fetchall() data = [] aux = [] for x in myresult: aux = x for y in aux: data.append(y) if session_wise: input_mmap_file_path = data[2] elif equalization: input_mmap_file_path = data[3] else: input_mmap_file_path = data[4] parameters = {'min_SNR': data[1], 'rval_thr': data[5], 'use_cnn': data[6]} data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/session_wise/' if \ data[0] else os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/trial_wise/' sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,equalization_v,source_extraction_v,component_evaluation_v,input,home_path,decoding_main FROM Analysis WHERE source_extraction_main=?" val = [ input_file, ] mycursor.execute(sql, val) result = mycursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[10] == 0: data[10] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}" output_file_path = f'main/{file_name}.hdf5' sql1 = "UPDATE Analysis SET component_evaluation_main=?,component_evaluation_v=? WHERE source_extraction_main=? " val1 = [output_file_path, data[10], input_file] mycursor.execute(sql1, val1) else: data[10] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}" output_file_path = f'main/{file_name}.hdf5' sql2 = "INSERT INTO Analysis (component_evaluation_main,component_evaluation_v) VALUES (?,?)" val2 = [output_file_path, data[10]] mycursor.execute(sql2, val2) database.commit() output_file_path_full = data_dir + output_file_path # Load CNMF object cnm = load_CNMF(input_mmap_file_path) # Load the original movie Yr, dims, T = cm.load_memmap(input_mmap_file_path) images = Yr.T.reshape((T, ) + dims, order='F') # Set the parmeters cnm.params.set('quality', parameters) # Stop the cluster if one exists n_processes = psutil.cpu_count() try: cm.cluster.stop_server() except: pass # Start a new cluster c, dview, n_processes = cm.cluster.setup_cluster( backend='local', n_processes=n_processes, # number of process to use, if you go out of memory try to reduce this one single_thread=False) # Evaluate components cnm.estimates.evaluate_components(images, cnm.params, dview=dview) logging.debug('Number of total components: ', len(cnm.estimates.C)) logging.debug('Number of accepted components: ', len(cnm.estimates.idx_components)) # Stop the cluster dview.terminate() # Save CNMF object cnm.save(output_file_path_full) return output_file_path
def run_alignment(mouse, sessions, motion_correction_v, cropping_v, dview): """ This is the main function for the alignment step. It applies methods from the CaImAn package used originally in motion correction to do alignment. """ for session in sessions: # Update the database file_name = f"mouse_{mouse}_session_{session}_alignment" sql1 = "UPDATE Analysis SET alignment_main=? WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? " val1 = [file_name, mouse, session, motion_correction_v, cropping_v] cursor.execute(sql1, val1) # Determine the output .mmap file name output_mmap_file_path = os.environ[ 'DATA_DIR_LOCAL'] + f'data/interim/alignment/main/{file_name}.mmap' sql = "SELECT motion_correction_main FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? " val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() input_mmap_file_list = [] inter = [] for x in result: inter += x for y in inter: input_mmap_file_list.append(y) sql = "SELECT motion_correction_cropping_points_x1 FROM Analysis WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? " val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() x_ = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): x_.append(inter[j]) sql = "SELECT motion_correction_cropping_points_x2 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? " val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() _x = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): _x.append(inter[j]) sql = "SELECT motion_correction_cropping_points_y1 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() _y = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): _y.append(inter[j]) sql = "SELECT motion_correction_cropping_points_y2 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() y_ = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): y_.append(inter[j]) new_x1 = max(x_) new_x2 = max(_x) new_y1 = max(y_) new_y2 = max(_y) m_list = [] for i in range(len(input_mmap_file_list)): m = cm.load(input_mmap_file_list[i]) m = m.crop(new_x1 - x_[i], new_x2 - _x[i], new_y1 - y_[i], new_y2 - _y[i], 0, 0) m_list.append(m) # Concatenate them using the concat function m_concat = cm.concatenate(m_list, axis=0) fname = m_concat.save(output_mmap_file_path, order='C') # MOTION CORRECTING EACH INDIVIDUAL MOVIE WITH RESPECT TO A TEMPLATE MADE OF THE FIRST MOVIE logging.info( 'Performing motion correction on all movies with respect to a template made of the first movie.' ) t0 = datetime.datetime.today() # parameters alignment sql5 = "SELECT make_template_from_trial,gSig_filt,max_shifts,niter_rig,strides,overlaps,upsample_factor_grid,num_frames_split,max_deviation_rigid,shifts_opencv,use_conda,nonneg_movie, border_nan FROM Analysis WHERE alignment_main=? " val5 = [ file_name, ] cursor.execute(sql5, val5) myresult = cursor.fetchall() para = [] aux = [] for x in myresult: aux = x for y in aux: para.append(y) parameters = { 'make_template_from_trial': para[0], 'gSig_filt': (para[1], para[1]), 'max_shifts': (para[2], para[2]), 'niter_rig': para[3], 'strides': (para[4], para[4]), 'overlaps': (para[5], para[5]), 'upsample_factor_grid': para[6], 'num_frames_split': para[7], 'max_deviation_rigid': para[8], 'shifts_opencv': para[9], 'use_cuda': para[10], 'nonneg_movie': para[11], 'border_nan': para[12] } # Create a template of the first movie template_index = parameters['make_template_from_trial'] m0 = cm.load(input_mmap_file_list[1]) [x1, x2, y1, y2] = [x_, _x, y_, _y] for i in range(len(input_mmap_file_list)): m0 = m0.crop(new_x1 - x_[i], new_x2 - _x[i], new_y1 - y_[i], new_y2 - _y[i], 0, 0) m0_filt = cm.movie( np.array([ high_pass_filter_space(m_, parameters['gSig_filt']) for m_ in m0 ])) template0 = cm.motion_correction.bin_median( m0_filt.motion_correct( 5, 5, template=None)[0]) # may be improved in the future # Setting the parameters opts = params.CNMFParams(params_dict=parameters) # Create a motion correction object mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion')) # Perform non-rigid motion correction mc.motion_correct(template=template0, save_movie=True) # Cropping borders x_ = math.ceil( abs(np.array(mc.shifts_rig)[:, 1].max() ) if np.array(mc.shifts_rig)[:, 1].max() > 0 else 0) _x = math.ceil( abs(np.array(mc.shifts_rig)[:, 1].min() ) if np.array(mc.shifts_rig)[:, 1].min() < 0 else 0) y_ = math.ceil( abs(np.array(mc.shifts_rig)[:, 0].max() ) if np.array(mc.shifts_rig)[:, 0].max() > 0 else 0) _y = math.ceil( abs(np.array(mc.shifts_rig)[:, 0].min() ) if np.array(mc.shifts_rig)[:, 0].min() < 0 else 0) # Load the motion corrected movie into memory movie = cm.load(mc.fname_tot_rig[0]) # Crop all movies to those border pixels movie.crop(x_, _x, y_, _y, 0, 0) sql1 = "UPDATE Analysis SET alignment_x1=?, alignment_x2 =?, alignment_y1=?, alignment_y2=? WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val1 = [ x_, _x, y_, _y, mouse, session, motion_correction_v, cropping_v ] cursor.execute(sql1, val1) # save motion corrected and cropped movie output_mmap_file_path_tot = movie.save( os.environ['DATA_DIR_LOCAL'] + f'data/interim/alignment/main/{file_name}.mmap', order='C') logging.info( f' Cropped and saved rigid movie as {output_mmap_file_path_tot}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) # Create a timeline and store it sql = "SELECT trial FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?" val = [mouse, session, motion_correction_v, cropping_v] cursor.execute(sql, val) result = cursor.fetchall() trial_index_list = [] inter = [] for i in result: inter += i for j in range(0, len(inter)): trial_index_list.append(inter[j]) timeline = [[trial_index_list[0], 0]] timepoints = [0] for i in range(1, len(m_list)): m = m_list[i] timeline.append( [trial_index_list[i], timeline[i - 1][1] + m.shape[0]]) timepoints.append(timepoints[i - 1] + m.shape[0]) timeline_pkl_file_path = os.environ[ 'DATA_DIR'] + f'data/interim/alignment/meta/timeline/{file_name}.pkl' with open(timeline_pkl_file_path, 'wb') as f: pickle.dump(timeline, f) sql1 = "UPDATE Analysis SET alignment_timeline=? WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? " val1 = [ timeline_pkl_file_path, mouse, session, motion_correction_v, cropping_v ] cursor.execute(sql1, val1) timepoints.append(movie.shape[0]) dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes sql1 = "UPDATE Analysis SET alignment_duration_concatenation=? WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? " val1 = [dt, mouse, session, motion_correction_v, cropping_v] cursor.execute(sql1, val1) logging.info(f' Performed concatenation. dt = {dt} min.') ## modify all motion correction file to the aligned version data_dir = os.environ[ 'DATA_DIR'] + 'data/interim/motion_correction/main/' for i in range(len(input_mmap_file_list)): aligned_movie = movie[timepoints[i]:timepoints[i + 1]] motion_correction_output_aligned = aligned_movie.save( data_dir + file_name + '_els' + '.mmap', order='C') sql1 = "UPDATE Analysis SET motion_correct_align=? WHERE motion_correction_meta=? AND motion_correction_v" val1 = [ motion_correction_output_aligned, input_mmap_file_list[i], motion_correction_v ] cursor.execute(sql1, val1) database.commit() return
def run_equalizer(input_file, session_wise=False): """ This function is meant to help with differences in contrast in different trials and session, to equalize general brightness or reduce photobleaching. It corrects the video and saves them in the corrected version. It can be run with the already aligned videos or trial by trial. for trial by trial, a template is required. """ # Take all the parameters needed for equalization sql5 = "SELECT make_template_from_trial,equalizer,histogram_step FROM Analysis WHERE alignment_main=? OR motion_correction_main =? " val5 = [input_file, input_file] cursor.execute(sql5, val5) myresult = cursor.fetchall() para = [] aux = [] for x in myresult: aux = x for y in aux: para.append(y) parameters = { 'make_template_from_trial': para[0], 'equalizer': para[1], 'histogram_step': para[2] } # determine the output file output_tif_file_path = os.environ[ 'DATA_DIR'] + f'data/interim/equalizer/main/' #determine the file name sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,equalization_v,input,home_path,decoding_main FROM Analysis WHERE alignment_main =? Or motion_correction_main =?" val = [input_file, input_file] cursor.execute(sql, val) result = cursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[8] == 0: data[8] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}" sql1 = "UPDATE Analysis SET equalization_main=?,equalization_v=? WHERE alignment_main=? " val1 = [file_name, data[8], input_file] cursor.execute(sql1, val1) else: data[8] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}" sql2 = "INSERT INTO Analysis (motion_correction_meta,motion_correction_v) VALUES (?,?)" val2 = [file_name, data[8]] cursor.execute(sql2, val2) database.commit() database.commit() input_tif_file_list = os.path.join(os.environ['DATA_DIR_LOCAL'], input_file) movie_original = cm.load( input_tif_file_list) # load video as 3d array already concatenated if parameters['make_template_from_trial'] == 0: movie_equalized = do_equalization(movie_original) else: movie_equalized = np.empty_like(movie_original) source = movie_original[0:100, :, :] # equalize all the videos loads in m_list_reshape with the histogram of source for j in range(int(movie_original.shape[0] / 100)): want_to_equalize = movie_original[j * 100:(j + 1) * 100, :, :] movie_equalized[j * 100:(j + 1) * 100, :, :] = do_equalization_from_template( reference=want_to_equalize, source=source) #Save the movie equalized_path = movie_equalized.save(output_tif_file_path + file_name + '.mmap', order='C') return output_tif_file_path
def run_motion_correction(cropping_file, dview): """ This is the function for motion correction. Its goal is to take in a decoded and cropped .tif file, perform motion correction, and save the result as a .mmap file. This function is only runnable on the cn76 server because it requires parallel processing. Args: cropping_file: tif file after cropping dview: cluster Returns: row: pd.DataFrame object The row corresponding to the motion corrected analysis state. """ # Get output file paths data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/motion_correction/' sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,input,home_path,decoding_main FROM Analysis WHERE cropping_main=? ORDER BY motion_correction_v" val = [ cropping_file, ] cursor.execute(sql, val) result = cursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[6] == 0: data[6] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}" output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl' sql1 = "UPDATE Analysis SET motion_correction_meta=?,motion_correction_v=? WHERE cropping_main=? " val1 = [output_meta_pkl_file_path, data[6], cropping_file] cursor.execute(sql1, val1) else: data[6] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}" output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl' sql2 = "INSERT INTO Analysis (motion_correction_meta,motion_correction_v) VALUES (?,?)" val2 = [output_meta_pkl_file_path, data[6]] cursor.execute(sql2, val2) database.commit() sql3 = "UPDATE Analysis SET decoding_main=?,decoding_v=?,mouse=?,session=?,trial=?,is_rest=?,input=?,home_path=?,cropping_v=?,cropping_main=? WHERE motion_correction_meta=? AND motion_correction_v=?" val3 = [ data[9], data[4], data[0], data[1], data[2], data[3], data[7], data[8], data[5], cropping_file, output_meta_pkl_file_path, data[6] ] cursor.execute(sql3, val3) database.commit() output_meta_pkl_file_path_full = data_dir + output_meta_pkl_file_path # Calculate movie minimum to subtract from movie cropping_file_full = os.environ['DATA_DIR_LOCAL'] + cropping_file min_mov = np.min(cm.load(cropping_file_full)) # Apply the parameters to the CaImAn algorithm sql5 = "SELECT motion_correct,pw_rigid,save_movie_rig,gSig_filt,max_shifts,niter_rig,strides,overlaps,upsample_factor_grid,num_frames_split,max_deviation_rigid,shifts_opencv,use_conda,nonneg_movie, border_nan FROM Analysis WHERE cropping_main=? " val5 = [ cropping_file, ] cursor.execute(sql5, val5) myresult = cursor.fetchall() para = [] aux = [] for x in myresult: aux = x for y in aux: para.append(y) parameters = { 'motion_correct': para[0], 'pw_rigid': para[1], 'save_movie_rig': para[2], 'gSig_filt': (para[3], para[3]), 'max_shifts': (para[4], para[4]), 'niter_rig': para[5], 'strides': (para[6], para[6]), 'overlaps': (para[7], para[7]), 'upsample_factor_grid': para[8], 'num_frames_split': para[9], 'max_deviation_rigid': para[10], 'shifts_opencv': para[11], 'use_cuda': para[12], 'nonneg_movie': para[13], 'border_nan': para[14] } caiman_parameters = parameters.copy() caiman_parameters['min_mov'] = min_mov opts = params.CNMFParams(params_dict=caiman_parameters) # Rigid motion correction (in both cases) logging.info('Performing rigid motion correction') t0 = datetime.datetime.today() # Create a MotionCorrect object mc = MotionCorrect([cropping_file_full], dview=dview, **opts.get_group('motion')) # Perform rigid motion correction mc.motion_correct_rigid(save_movie=parameters['save_movie_rig'], template=None) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes logging.info(f' Rigid motion correction finished. dt = {dt} min') # Obtain template, rigid shifts and border pixels total_template_rig = mc.total_template_rig shifts_rig = mc.shifts_rig # Save template, rigid shifts and border pixels in a dictionary meta_pkl_dict = { 'rigid': { 'template': total_template_rig, 'shifts': shifts_rig, } } sql = "UPDATE Analysis SET duration_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [dt, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) if parameters['save_movie_rig'] == 1: # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f' Loading rigid movie for cropping') m_rig = cm.load(mc.fname_tot_rig[0]) logging.info(f' Loaded rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_rigid_shifts(shifts_rig) # Crop the movie logging.info( f' Cropping and saving rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_rig = m_rig.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['rigid']['cropping_points'] = [x_, _x, y_, _y] sql = "UPDATE Analysis SET motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [x_, _x, y_, _y, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) # Save the movie rig_role = 'alternate' if parameters['pw_rigid'] else 'main' fname_tot_rig = m_rig.save(data_dir + rig_role + '/' + file_name + '_rig' + '.mmap', order='C') logging.info(f' Cropped and saved rigid movie as {fname_tot_rig}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_rig[0]) sql = "UPDATE Analysis SET motion_correction_rig_role=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [fname_tot_rig, output_meta_pkl_file_path, data[6]] cursor.execute(sql, val) database.commit() # If specified in the parameters, apply piecewise-rigid motion correction if parameters['pw_rigid'] == 1: logging.info(f' Performing piecewise-rigid motion correction') t0 = datetime.datetime.today() # Perform non-rigid (piecewise rigid) motion correction. Use the rigid result as a template. mc.motion_correct_pwrigid(save_movie=True, template=total_template_rig) # Obtain template and filename total_template_els = mc.total_template_els fname_tot_els = mc.fname_tot_els[0] dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes meta_pkl_dict['pw_rigid'] = { 'template': total_template_els, 'x_shifts': mc.x_shifts_els, 'y_shifts': mc. y_shifts_els # removed them initially because they take up space probably } logging.info( f' Piecewise-rigid motion correction finished. dt = {dt} min') # Load the movie saved by CaImAn, which is in the wrong # directory and is not yet cropped logging.info(f' Loading pw-rigid movie for cropping') m_els = cm.load(fname_tot_els) logging.info(f' Loaded pw-rigid movie for cropping') # Get the cropping points determined by the maximal rigid shifts x_, _x, y_, _y = get_crop_from_pw_rigid_shifts( np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) # Crop the movie logging.info( f' Cropping and saving pw-rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}' ) m_els = m_els.crop(x_, _x, y_, _y, 0, 0) meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y] # Save the movie fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C') logging.info(f'Cropped and saved rigid movie as {fname_tot_els}') # Remove the remaining non-cropped movie os.remove(mc.fname_tot_els[0]) sql = "UPDATE Analysis SET motion_correction_main=?, motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=?,duration_pw_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? " val = [ fname_tot_els, x_, _x, y_, _y, dt, output_meta_pkl_file_path, data[6] ] cursor.execute(sql, val) database.commit() # Write meta results dictionary to the pkl file pkl_file = open(output_meta_pkl_file_path_full, 'wb') pickle.dump(meta_pkl_dict, pkl_file) pkl_file.close() return fname_tot_els, data[6]
def run_registration(input_file): """ This is the main registering function. Is is supposed to be run after trial wise component evaluation. Registration takes over different contours of trial wise source extracted contours and do a matching between cells. It can use two different methods: Hungarian matching algorithm (RegisterMulti) (as implement in Giovannucci, et al. 2019) or cell registration (CellReg)using centroids distance and spatial correlation (as implemented in Sheintuch, et al. 2017). Default method is registration with no modeling of distributions of centroids and spatial correlation. """ sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,source_extraction_v,equalization_v,component_evaluation_v,registration_v FROM Analysis WHERE component_evaluation_main=?" val = [ input_file, ] cursor.execute(sql, val) result = cursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[11] == 0: data[11] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[9]}.{data[8]}.{data[10]}.{data[11]}" sql1 = "UPDATE Analysis SET motion_correction_meta=?,motion_correction_v=? WHERE cropping_main=? " val1 = [file_name, data[11], input_file] cursor.execute(sql1, val1) else: data[6] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}" sql2 = "INSERT INTO Analysis (motion_correction_meta,motion_correction_v) VALUES (?,?)" val2 = [file_name, data[11]] cursor.execute(sql2, val2) database.commit() database.commit() if parameters['session_wise'] == False: data_dir = os.environ[ 'DATA_DIR'] + 'data/interim/registration/trial_wise/main/' else: data_dir = os.environ[ 'DATA_DIR'] + 'data/interim/registration/session_wise/main/' file_name = db.create_file_name(step_index, row_new.name) output_file_path = data_dir + f'{file_name}.pkl' ##create the dictionary with metadata information output = { 'main': output_file_path, 'meta': { 'analysis': { 'analyst': os.environ['ANALYST'], 'date': datetime.datetime.today().strftime("%m-%d-%Y"), 'time': datetime.datetime.today().strftime("%H:%M:%S") }, 'duration': {} } } ## take alignment data for the timeline of alingment first_row = df.iloc[0] alignmnet_output = eval(first_row['alignment_output']) alignment_timeline_file = alignmnet_output['meta']['timeline'] ## multiple list created to append the relevant information for the registration and creation of a unique time trace ## matrix (cnm.estimates.A and cnm.estimates.C ) both taken after component evaluation A_list = [] ## list for contour matrix on multiple trials #A_size = [] ## list for the size of A (just to verify it is always the same size) FOV_size = [ ] ## list for the cn filter dim (to verify it is always the same dims) A_number_components = [ ] ## list with the total number of components extracted for each trial C_dims = [] ## dimension of C, to keep track of timeline C_list = [] ## list with traces for each trial evaluated_trials = [] evaluated_session = [] typical_size = [] for i in range(len(df)): row = df.iloc[i] component_evaluation_hdf5_file_path = eval( row['component_evaluation_output'])['main'] corr_path = eval( row['source_extraction_output'])['meta']['corr']['main'] cnm = load_CNMF(component_evaluation_hdf5_file_path) cn_filter = np.load(db.get_file(corr_path)) FOV_size.append(cn_filter.shape) #A_size.append(cnm.estimates.A.shape[0]) A_number_components.append(cnm.estimates.idx_components.shape[0]) A_list.append(cnm.estimates.A[:, cnm.estimates.idx_components]) C_dims.append(cnm.estimates.C.shape) size = cnm.estimates.A[:, cnm.estimates.idx_components].sum(axis=0) for j in range(len(cnm.estimates.idx_components)): typical_size.append(size[0, j]) if cnm.estimates.bl is None: C_list.append(cnm.estimates.C[cnm.estimates.idx_components, :]) else: C_list.append(cnm.estimates.C[cnm.estimates.idx_components, :] - cnm.estimates.bl[cnm.estimates.idx_components, np.newaxis]) evaluated_trials.append( (df.iloc[i].name[2] - 1) * 2 + df.iloc[i].name[3]) ## number that goes from 0 to 42 evaluated_session.append(df.iloc[i].name[1]) ## add a size restriction on the neurons that will further be processed. This restriction boundary # decision is based in the histogram of typical neuronal sizes min_size = parameters['min_cell_size'] max_size = parameters['max_cell_size'] new_A_list = [] new_C_list = [] A_components = [] C_dims_new = [] new_evaluated_trials = [] new_evaluated_session = [] for i in range(len(A_list)): accepted_size = [] size = A_list[i].sum(axis=0) for j in range(size.shape[1]): if size[0, j] > 10 and size[0, j] < 25: accepted_size.append(j) if len(accepted_size) > 1: new_A_list.append(A_list[i][:, accepted_size]) new_C_list.append(C_list[i][accepted_size, :]) A_components.append(A_number_components[i]) C_dims_new.append(new_C_list[-1].shape) new_evaluated_trials.append(evaluated_trials[i]) new_evaluated_session.append(evaluated_session[i]) A_list = new_A_list C_list = new_C_list ## run CaImAn registration rutine that use the Hungarian matching algorithm in the contours list spatial_union, assignments, match = register_multisession( A=A_list, dims=FOV_size[0], thresh_cost=parameters['cost_threshold'], max_dist=parameters['max_dist']) ## open the timeline and create the new traces matrix C_matrix with open(alignment_timeline_file, 'rb') as f: timeline = pickle.load(f) total_time = timeline[len(timeline) - 1][1] + C_list[len(C_list) - 1].shape[1] timeline.append(['End', total_time]) C_matrix = np.zeros((spatial_union.shape[1], total_time)) new_assignments = np.zeros((spatial_union.shape[1], len(timeline))) for i in range(spatial_union.shape[1]): for j in range(assignments.shape[1]): trial = new_evaluated_trials[j] if math.isnan(assignments[i, j]) == False: new_assignments[i][trial] = assignments[i, j] + 1 unique_session = [] for x in evaluated_session: if x not in unique_session: unique_session.append(x) session_vector = np.arange(0, len(unique_session)) final_evaluated_session = [] for i in range(assignments.shape[1]): for j in range(len(unique_session)): if new_evaluated_session[i] == unique_session[j]: final_evaluated_session.append(session_vector[j]) for i in range(spatial_union.shape[1]): for j in range(assignments.shape[1]): trial = (final_evaluated_session[j] + 1) * new_evaluated_trials[j] print(trial) if math.isnan(assignments[i, j]) == False: C_matrix[i][timeline[trial][1]:timeline[trial][1] + C_dims_new[j][1]] = ( C_list[j])[int(assignments[i, j]), :] cnm_registration = estimates(A=spatial_union, C=C_matrix) with open(output_file_path, 'wb') as output_file: pickle.dump(cnm_registration, output_file, pickle.HIGHEST_PROTOCOL) return
def run_decoder(mouse, session, trial): """ This is the function for the decoding step. In the decoding step files are converted from .raw files to .tif files. This function is only usable on the Sebastian's account on the pastiera pc. Args: mouse, session, trial, is_rest: parameters of interest Returns: path of the decoded file """ sql = "SELECT input, home_path, is_rest FROM Analysis WHERE mouse =? AND session = ? AND trial = ?" val = (mouse, session, trial) mycursor.execute(sql, val) result = mycursor.fetchall() input_raw_file = [] for row in result: input_raw_file += row input_raw_file_paths = input_raw_file[1] + input_raw_file[0] + '.raw' # create the correct name for the file file_name = f"mouse_{mouse}_session_{session}_trial_{trial}_{input_raw_file[2]}.v1" output_tif_file_path = f"data/interim/decoding/main/{file_name}.tif" # Decoder paths py_inscopix = '/home/morgane/anaconda3/envs/inscopix_reader/bin/python' decoder = "/home/morgane/src/inscopix_reader_linux/python/downsampler.py" # Decoding print('Performing decoding on raw file') # Convert the output tif file path to the full path such that the downsampler.py script can use them. output_tif_file_path_full = os.path.join(os.environ['DATA_DIR_LOCAL'], output_tif_file_path) # Make a command usable by the decoder script (downsampler.py, see the script for more info) input_xml_file_path = input_raw_file[1] + input_raw_file[0] + '.xml' cmd = ' '.join([ py_inscopix, decoder, '"' + input_raw_file_paths + '"', output_tif_file_path_full, '"' + input_xml_file_path + '"' ]) # Run the command subprocess.check_output(cmd, shell=True) print('Decoding finished') sql1 = "UPDATE Analysis SET decoding_v = ?, decoding_main= ? WHERE mouse = ? AND session = ? AND trial = ? AND is_rest = ?" val1 = (1, output_tif_file_path, mouse, session, trial, input_raw_file[2]) mycursor.execute(sql1, val1) database.commit() return output_tif_file_path
def run_source_extraction(input_file, dview): """ This is the function for source extraction. Its goal is to take in a .mmap file, perform source extraction on it using cnmf-e and save the cnmf object as a .pkl file. """ sql = "SELECT equalization,source_extraction_session_wise,fr,decay_time,min_corr,min_pnr,p,K,gSig,merge_thr,rf,stride,tsub,ssub,p_tsub,p_ssub,low_rank_background,nb,nb_patch,ssub_B,init_iter,ring_size_factor,method_init,method_deconvolution,update_background_components,center_psf,border_pix,normalize_init,del_duplicates,only_init FROM Analysis WHERE motion_correction_main =? OR alignment_main = ? OR equalization_main =?" val = [input_file, input_file, input_file] cursor.execute(sql, val) result = cursor.fetchall() para = [] inter = [] for x in result: inter = x for y in inter: para.append(y) gSiz = 4 * para[8] + 1 parameters = { 'equalization': para[0], 'session_wise': para[1], 'fr': para[2], 'decay_time': para[3], 'min_corr': para[4], 'min_pnr': para[5], 'p': para[6], 'K': para[7], 'gSig': (para[8], para[8]), 'gSiz': (gSiz, gSiz), 'merge_thr': para[9], 'rf': para[10], 'stride': para[11], 'tsub': para[12], 'ssub': para[13], 'p_tsub': para[14], 'p_ssub': para[15], 'low_rank_background': para[16], 'nb': para[17], 'nb_patch': para[18], 'ssub_B': para[19], 'init_iter': para[20], 'ring_size_factor': para[21], 'method_init': para[22], 'method_deconvolution': para[23], 'update_background_components': para[24], 'center_psf': para[25], 'border_pix': para[26], 'normalize_init': para[27], 'del_duplicates': para[28], 'only_init': para[29] } # Determine output paths if parameters['session_wise']: data_dir = os.environ[ 'DATA_DIR_LOCAL'] + 'data/interim/source_extraction/session_wise/' else: data_dir = os.environ[ 'DATA_DIR_LOCAL'] + 'data/interim/source_extraction/trial_wise/' sql1 = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,equalization_v,source_extraction_v,input,home_path,decoding_main FROM Analysis WHERE motion_correction_main =? OR alignment_main = ? OR equalization_main =?" val1 = [input_file, input_file, input_file] cursor.execute(sql1, val1) result = cursor.fetchall() data = [] inter = [] for x in result: inter = x for y in inter: data.append(y) # Update the database if data[9] == 0: data[9] = 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}" output_file_path = data_dir + f'main/{file_name}.hdf5' sql1 = "UPDATE Analysis SET source_extraction_main=?,source_extraction_v=? WHERE motion_correction_main =? OR alignment_main = ? OR equalization_main =? " val1 = [output_file_path, data[9], input_file, input_file, input_file] cursor.execute(sql1, val1) else: data[9] += 1 file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}" output_file_path = data_dir + f'main/{file_name}.hdf5' sql2 = "INSERT INTO Analysis (source_extraction_main,source_extraction_v) VALUES (?,?)" val2 = [output_file_path, data[10]] cursor.execute(sql2, val2) database.commit() database.commit() # Load memmory mappable input file if os.path.isfile(input_file): Yr, dims, T = cm.load_memmap(input_file) images = Yr.T.reshape((T, ) + dims, order='F') else: logging.warning(f' .mmap file does not exist. Cancelling') # SOURCE EXTRACTION # Check if the summary images are already there corr_npy_file_path, pnr_npy_file_path = get_corr_pnr_path( gSig_abs=parameters['gSig'][0]) if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path): # Already computed summary images logging.info(f' Already computed summary images') cn_filter = np.load(corr_npy_file_path) pnr = np.load(pnr_npy_file_path) else: # Compute summary images t0 = datetime.datetime.today() logging.info(f' Computing summary images') cn_filter, pnr = cm.summary_images.correlation_pnr( images[::1], gSig=parameters['gSig'][0], swap_dim=False) dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes logging.info(f' Computed summary images. dt = {dt} min') # Saving summary images as npy files gSig = parameters['gSig'][0] corr_npy_file_path = data_dir + f'/meta/corr/{file_name}_gSig_{gSig}.npy' pnr_npy_file_path = data_dir + f'/meta/pnr/{file_name}_gSig_{gSig}.npy' with open(corr_npy_file_path, 'wb') as f: np.save(f, cn_filter) with open(pnr_npy_file_path, 'wb') as f: np.save(f, pnr) # Calculate min, mean, max value for cn_filter and pnr corr_min, corr_mean, corr_max = cn_filter.min(), cn_filter.mean( ), cn_filter.max() pnr_min, pnr_mean, pnr_max = pnr.min(), pnr.mean(), pnr.max() # If min_corr and min_pnr are specified via a linear equation, calculate # this value if type(parameters['min_corr']) == list: min_corr = parameters['min_corr'][0] * corr_mean + parameters[ 'min_corr'][1] parameters['min_corr'] = min_corr logging.info(f' Automatically setting min_corr = {min_corr}') if type(parameters['min_pnr']) == list: min_pnr = parameters['min_pnr'][0] * pnr_mean + parameters['min_pnr'][1] parameters['min_pnr'] = min_pnr logging.info(f' Automatically setting min_pnr = {min_pnr}') # Set the parameters for caiman opts = params.CNMFParams(params_dict=parameters) # SOURCE EXTRACTION logging.info(f' Performing source extraction') t0 = datetime.datetime.today() n_processes = psutil.cpu_count() logging.info(f' n_processes: {n_processes}') cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, params=opts) cnm.fit(images) cnm.estimates.dims = dims # Calculate the center of masses cnm.estimates.center = caiman.base.rois.com(cnm.estimates.A, images.shape[1], images.shape[2]) # Save the cnmf object as a hdf5 file logging.info(f' Saving cnmf object') cnm.save(output_file_path) dt = int( (datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes logging.info(f' Source extraction finished. dt = {dt} min') sql1 = "UPDATE Analysis SET duration_summary_images=?,source_extraction_corr=?, source_extraction_pnr=?, source_extraction_corr_min =?, source_extraction_corr_mean=?, source_extraction_corr_max=?, source_extraction_pnr_min=?,source_extraction_pnr_mean=?,source_extraction_pnr_max=?,source_extraction_k=?,source_extraction_duration=?,min_corr=?,min_pnr=? WHERE source_extraction_main= ? AND source_extraction_v=? " val1 = [ dt, corr_npy_file_path, pnr_npy_file_path, corr_min, corr_mean, corr_max, pnr_min, pnr_mean, pnr_max, len(cnm.estimates.C), dt, output_file_path, data[9] ] cursor.execute(sql1, val1) return output_file_path, data[9]
def run_cropper(input_path): ''' This function takes in a decoded analysis state and crops it according to specified cropping points. Args: input_path the path of the decoding file Returns row: pd.DataFrame object The row corresponding to the cropped analysis state. ''' # Determine output .tif file path sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v FROM Analysis WHERE decoding_main=%s ORDER BY cropping_v" val = [ input_path, ] mycursor.execute(sql, val) myresult = mycursor.fetchall() data = [] for x in myresult: data += x file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[5]}.{data[4]}" output_tif_file_path = f"data/interim/cropping/main/{file_name}.tif" # update the database if data[4] == 0: data[4] = 1 sql1 = "UPDATE Analysis SET cropping_main=%s,cropping_v=%s WHERE decoding_main=%s " val1 = [output_tif_file_path, data[4], input_path] mycursor.execute(sql1, val1) database.commit() else: data[4] += 1 sql2 = "INSERT INTO Analysis (cropping_main,cropping_v) VALUES (%s,%s)" val2 = [output_tif_file_path, data[4]] mycursor.execute(sql2, val2) database.commit() sql3 = "UPDATE Analysis SET decoding_main=%s,decoding_v=%s,mouse=%s,session=%s,trial=%s,is_rest=%s WHERE cropping_main=%s AND cropping_v=%s" val3 = [ input_path, data[5], data[0], data[1], data[2], data[3], output_tif_file_path, data[4] ] mycursor.execute(sql3, val3) database.commit() # Spatial copping logging.info('Loading movie') m = cm.load(input_path) logging.info('Loaded movie') # Choose crop parameters x1 = int(input("Limit X1 : ")) x2 = int(input("Limit X2 : ")) y1 = int(input("Limit Y1 : ")) y2 = int(input("Limit Y2 : ")) sql = "INSERT INTO Analysis (crop_spatial,cropping_points_spatial,crop_temporal,cropping_points_temporal) VALUES (%s,%s,%s,%s) WHERE cropping_main=%s AND cropping_v=%s" val = [True, [y1, y2, x1, x2], False, [], output_tif_file_path, data[4]] mycursor.execute(sql, val) database.commit() [x_, _x, y_, _y] = [y1, y2, x1, x2] logging.info('Performing spatial cropping') m = m[:, x_:_x, y_:_y] logging.info(' Spatial cropping finished') # Save the movie m.save(output_tif_file_path) return output_tif_file_path