def run_subsampling(): for dataset in ['4x4', '5x5']: basepath = '/data.nst/jdehning/packer_data/calcium_subsampled/' dic_prop = { '2x1': [ '2019-11-08_RL065_t-001_Cycle00001_Ch3_2x1.hdf5', '2019-11-08_RL065_t-001_2x1', 16 ], '2x2': [ '2019-11-08_RL065_t-001_Cycle00001_Ch3_2x2.hdf5', '2019-11-08_RL065_t-001_2x2', 13 ], '3x3': [ '2019-11-08_RL065_t-001_Cycle00001_Ch3_3x3.hdf5', '2019-11-08_RL065_t-001_3x3', 9 ], '1x2': [ '2019-11-08_RL065_t-001_Cycle00001_Ch3_1x2.hdf5', '2019-11-08_RL065_t-001_1x2', 9 ], '4x4': [ '2019-11-08_RL065_t-001_Cycle00001_Ch3_4x4.hdf5', '2019-11-08_RL065_t-001_4x4', 6 ], '5x5': [ '2019-11-08_RL065_t-001_Cycle00001_Ch3_5x5.hdf5', '2019-11-08_RL065_t-001_5x5', 5 ] } ops['diameter'] = dic_prop[dataset][2] db['h5py'] = basepath + dic_prop[dataset][0] db['save_path0'] = basepath + dic_prop[dataset][1] run_s2p(ops=ops, db=db)
def run_s2p_on(path, ops_file=None, reprocess=False, infer_from_recording_classes=False, inferrer=None): db = {'data_path': [path]} if ops_file == None: ops_file = os.path.join(path, "suite2p", "plane0", "ops.npy") try: ops = np.load(ops_file, allow_pickle=True).item() ops["keep_movie_raw"] #This is so we get a KeyError ops["keep_movie_raw"] = True ops["connected"] ops["connected"] = False ops["max_overlap"] ops["max_overlap"] = 0.2 ops["do_registration"] ops["do_registration"] = 2 #ALWAYS redo registration ops["look_one_level_down"] ops["look_one_level_down"] = True except FileNotFoundError: if all((any(["tif" in file for file in os.listdir(path)]), infer_from_recording_classes, inferrer != None)): #How many planes? no_of_planes = inferrer(exp_id(path)) if no_of_planes == 1: ops = default_ops() ops["nchannels"] = 2 ops["look_one_level_down"] = False ops["do_registration"] = 2 ops["keep_movie_raw"] = True ops["align_by_chan"] = 2 ops["nonrigid"] = False ops["connected"] = False ops["max_overlap"] = 0.2 ops["bidi_corrected"] = True ops["two_step_reigstration"] = True ops["sparse_mode"] = True try: run_s2p(ops=ops, db=db) except Exception as e: print( f"exception {e} raised at path {path} in response to run s2p call" ) print(f"db file:") for key, value in db.items(): print(f"{key}: {value}") print(f"ops file:") for key, value in ops.items(): print(f"{key}: {value}") else: print(f"No TIFFS at {path}" if infer_from_recording_classes else f"{path} not yet processed.") return else: if not reprocess: print(f"{path} has already been processed by suite2p") return run_s2p(ops=ops, db=db)
def single_file(f): head, tail = os.path.split(f) s2pexists = check4s2p(f) if s2pexists: if s2p_overwrite: shutil.rmtree(os.path.join(head, 'suite2p')) else: print("%s suite2p already done" % f) return None h5exists = check4h5(f) if not h5exists or h5_overwrite: if h5_output is None: h5fname = pp.sbx2h5(f, output_name=f + ".h5", max_idx=max_idx) else: h5fname = pp.sbx2h5(f, output_name=os.path.join( h5_output, tail + ".h5"), max_idx=max_idx) else: if h5_output is None: h5fname = f + '.h5' else: h5fname = os.path.join(h5_output, tail + ".h5") # set ops outpath = f try: os.makedirs(f) except: print("outpath not created") ops_d = {'save_path0': f} #ops_d['save_path0']=f ops = pp.set_ops(ops_d) # set db db = pp.set_db(h5fname, d=db_d) # run suite2p bindir = os.path.join(db['fast_disk'], 'suite2p') try: shutil.rmtree(bindir) except: pass run_s2p(ops=ops, db=db) # move registered binary over to save data folders if keep_binary: shutil.move(os.path.join(bindir, "plane0", "data.bin"), os.path.join(outpath, "suite2p", "data.bin")) # delete temporary files shutil.rmtree(bindir) return None
def run_for_single_file(animal_path, file): expID = get_expID(file) hf_path = os.path.join(animal_path, file) savepath = os.path.join(animal_path, file[:-3]) tmp_path = os.path.join(os.environ['SCRATCH'], 'register_jobs', 'bin_files', file[:-3]) mkdir_p(tmp_path) db = default_db() db['h5py'] = hf_path db['fast_disk'] = tmp_path ops = default_ops() ops['save_path0'] = savepath ops['fast_disk'] = tmp_path # run one experiment if not os.path.isdir(ops['save_path0']): start = time.time() print('Running for: ' + db['h5py'] + '\n') print('Saving on: ' + ops['save_path0']) print('Tmp saving:' + ops['fast_disk']) opsEnd = run_s2p(ops=ops, db=db) stop = time.time() timelog = open( os.path.join(os.environ['L_SCRATCH'], 'suite2p_times.txt'), 'a') timelog.write(db['h5py'] + ', ' + str(stop - start) + '\n') timelog.close() else: print(ops['save_path0'] + 'already exisits, skipping. \n') return
def run_suite2p(db, out_path, fs, nplanes): default = default_ops() default2 = { 'save_path0': out_path, 'delete_bin': False, 'look_one_level_down': True, 'data_path': db['in_dir'], 'nplanes': nplanes, 'fs': fs, 'save_mat': True, 'reg_tif': False, 'expts': db['expts'], 'raw': True, } if 'ops' in db: opts = db['ops'] else: opts = {} ops = {**default, **default2, **opts} db = {} print(ops) ops = run_s2p(ops=ops, db=db) return ops
def run_suite2p(hdf5_list, dirname_output, mdata): z_planes = mdata['size']['z_planes'] fs_param = 1. / (mdata['period'] * z_planes) # Load suite2p only right before use, as it has a long load time. from suite2p import run_s2p default_ops = run_s2p.default_ops() params = { 'input_format': 'h5', 'data_path': [str(f.parent) for f in hdf5_list], 'save_path0': str(dirname_output), 'nplanes': z_planes, 'fs': fs_param, 'save_mat': True, 'bidi_corrected': True, 'spatial_hp': 50, 'sparse_mode': False, 'threshold_scaling': 3, 'diameter': 6, } logger.info('Running suite2p on files:\n%s\n%s', '\n'.join(str(f) for f in hdf5_list), params) with open(dirname_output / 'recording_order.json', 'w') as fout: json.dump([str(e) for e in hdf5_list], fout, indent=4) run_s2p.run_s2p(ops=default_ops, db=params)
def process_data(animalid, date, expt_ids, raw_base='/home/mossing/data/suite2P/raw/', result_base='/home/mossing/data/suite2P/results/', fast_disk='/home/mossing/data_ssd/suite2P/bin', nchannels=1, delete_raw=False, diameter=15): # save_path0 = result_base+animalid+'/'+date+'/'+'_'.join(expt_ids) # data_path = [raw_base+animalid+'/'+date+'/'+lbl for lbl in expt_ids] db = prepare_db(animalid, date, expt_ids, raw_base=raw_base, result_base=result_base, fast_disk=fast_disk, nchannels=nchannels, diameter=diameter) # provide an h5 path in 'h5py' or a tiff path in 'data_path' # db overwrites any ops (allows for experiment specific settings) #db = { # 'h5py': [], # a single h5 file path # 'h5py_key': 'data', # 'look_one_level_down': False, # whether to look in ALL subfolders when searching for tiffs, # 'save_path0': save_path0, # 'data_path': data_path, # a list of folders with tiffs # # (or folder of folders with tiffs if look_one_level_down is True, or subfolders is not empty) # 'subfolders': [], # choose subfolders of 'data_path' to look in (optional) # 'fast_disk': fast_disk, # string which specifies where the binary file will be stored (should be an SSD) # 'nchannels': nchannels, # 'diameter': diameter # } try: shutil.rmtree(fast_disk + '/suite2p') print('fast disk contents deleted') except: print('fast disk location empty') opsEnd = run_s2p(ops=ops, db=db) if delete_raw: for fold in db['data_path']: for old_file in glob.glob(fold + '/*.tif'): os.remove(old_file)
def process_data_1ch_2ch(animalid, date, expt_ids_1ch, expt_ids_2ch, raw_base='/home/mossing/data/suite2P/raw/', result_base='/home/mossing/data/suite2P/results/', fast_disk='/home/mossing/data_ssd/suite2P/bin', delete_raw=False, diameter=15): # save_path0 = result_base+animalid+'/'+date+'/'+'_'.join(expt_ids) # data_path = [raw_base+animalid+'/'+date+'/'+lbl for lbl in expt_ids] # process 1ch data first db = prepare_db(animalid, date, expt_ids_1ch, raw_base=raw_base, result_base=result_base, fast_disk=fast_disk, nchannels=1, diameter=diameter, combined=False) try: shutil.rmtree(fast_disk + '/suite2p') print('fast disk contents deleted') except: print('fast disk location empty') # run suite2p part opsEnd = run_s2p(ops=ops, db=db) if delete_raw: for fold in db['data_path']: for old_file in glob.glob(fold + '/*.tif'): os.remove(old_file) add_2ch_data(animalid, date, expt_ids_1ch, expt_ids_2ch, raw_base=raw_base, result_base=result_base, fast_disk=fast_disk, delete_raw=delete_raw, diameter=diameter)
def s2pRun(self, user_batch_size): num_pixels = self.frame_x*self.frame_y sampling_rate = self.fps/self.n_planes diameter_x = 13/self.pix_sz_x diameter_y = 13/self.pix_sz_y diameter = int(diameter_x), int(diameter_y) batch_size = user_batch_size * (262144 / num_pixels) # larger frames will be more RAM intensive, scale user batch size based on num pixels in 512x512 images db = { 'data_path' : [self.tiff_path], 'fs' : float(sampling_rate), 'diameter' : diameter, 'batch_size' : int(batch_size), 'nimg_init' : int(batch_size), 'nplanes' : self.n_planes } print(db) opsEnd = run_s2p(ops=ops,db=db)
2, # number of pixels to keep between ROI and neuropil donut 'outer_neuropil_radius': np.inf, # maximum neuropil radius 'min_neuropil_pixels': 350, # minimum number of pixels in the neuropil 'high_pass': 100, # running mean subtraction with window of size 'high_pass' (use low values for 1P) # deconvolution settings 'baseline': 'maximin', # baselining mode 'win_baseline': 60., # window for maximin 'sig_baseline': 10., # smoothing constant for gaussian filter 'prctile_baseline': 8., # optional (whether to use a percentile baseline) 'neucoeff': .7, # neuropil coefficient } # provide an h5 path in 'h5py' or a tiff path in 'data_path' # db overwrites any ops (allows for experiment specific settings) db = { 'h5py': [], # a single h5 file path 'h5py_key': 'data', 'look_one_level_down': False, # whether to look in ALL subfolders when searching for tiffs 'data_path': [raw_dir], # a list of folders with tiffs # (or folder of folders with tiffs if look_one_level_down is True, or subfolders is not empty) 'subfolders': [], # choose subfolders of 'data_path' to look in (optional) 'fast_disk': scratch_folder, # string which specifies where the binary file will be stored (should be an SSD) # 'tiff_list': ['fov1_retino_00001.tif'] # list of tiffs in folder * data_path *! } # run one experiment opsEnd = run_s2p(ops=ops, db=db)
def find_rois_suite2p(video, video_path, params, mc_borders=None, use_multiprocessing=True): if suite2p_enabled: full_video_path = video_path directory = os.path.dirname(full_video_path) filename = os.path.basename(full_video_path) roi_spatial_footprints = [ None for i in range(video.shape[1]) ] roi_temporal_footprints = [ None for i in range(video.shape[1]) ] roi_temporal_residuals = [ None for i in range(video.shape[1]) ] bg_spatial_footprints = [ None for i in range(video.shape[1]) ] bg_temporal_footprints = [ None for i in range(video.shape[1]) ] if os.path.exists("suite2p"): shutil.rmtree("suite2p") for z in range(video.shape[1]): fname = os.path.splitext(filename)[0] + "_masked_z_{}.h5".format(z) video_path = os.path.join(directory, fname) h5f = h5py.File(video_path, 'w') h5f.create_dataset('data', data=video[:, z, :, :]) h5f.close() ops = { 'fast_disk': [], # used to store temporary binary file, defaults to save_path0 'save_path0': '', # stores results, defaults to first item in data_path 'delete_bin': False, # whether to delete binary file after processing # main settings 'nplanes' : 1, # each tiff has these many planes in sequence 'nchannels' : 1, # each tiff has these many channels per plane 'functional_chan' : 1, # this channel is used to extract functional ROIs (1-based) 'diameter':params['diameter'], # this is the main parameter for cell detection, 2-dimensional if Y and X are different (e.g. [6 12]) 'tau': 1., # this is the main parameter for deconvolution 'fs': params['sampling_rate'], # sampling rate (total across planes) # output settings 'save_mat': False, # whether to save output as matlab files 'combined': True, # combine multiple planes into a single result /single canvas for GUI # parallel settings 'num_workers': 0, # 0 to select num_cores, -1 to disable parallelism, N to enforce value 'num_workers_roi': 0, # 0 to select number of planes, -1 to disable parallelism, N to enforce value # registration settings 'do_registration': False, # whether to register data 'nimg_init': 200, # subsampled frames for finding reference image 'batch_size': 200, # number of frames per batch 'maxregshift': 0.1, # max allowed registration shift, as a fraction of frame max(width and height) 'align_by_chan' : 1, # when multi-channel, you can align by non-functional channel (1-based) 'reg_tif': False, # whether to save registered tiffs 'subpixel' : 10, # precision of subpixel registration (1/subpixel steps) # cell detection settings 'connected': params['connected'], # whether or not to keep ROIs fully connected (set to 0 for dendrites) 'navg_frames_svd': 5000, # max number of binned frames for the SVD 'nsvd_for_roi': 1000, # max number of SVD components to keep for ROI detection 'max_iterations': 20, # maximum number of iterations to do cell detection 'ratio_neuropil': params['neuropil_basis_ratio'], # ratio between neuropil basis size and cell radius 'ratio_neuropil_to_cell': params['neuropil_radius_ratio'], # minimum ratio between neuropil radius and cell radius 'tile_factor': 1., # use finer (>1) or coarser (<1) tiles for neuropil estimation during cell detection 'threshold_scaling': 1., # adjust the automatically determined threshold by this scalar multiplier 'max_overlap': 0.75, # cells with more overlap than this get removed during triage, before refinement 'inner_neuropil_radius': params['inner_neuropil_radius'], # number of pixels to keep between ROI and neuropil donut 'outer_neuropil_radius': np.inf, # maximum neuropil radius 'min_neuropil_pixels': params['min_neuropil_pixels'], # minimum number of pixels in the neuropil # deconvolution settings 'baseline': 'maximin', # baselining mode 'win_baseline': 60., # window for maximin 'sig_baseline': 10., # smoothing constant for gaussian filter 'prctile_baseline': 8.,# optional (whether to use a percentile baseline) 'neucoeff': .7, # neuropil coefficient } db = { 'h5py': video_path, # a single h5 file path 'h5py_key': 'data', 'look_one_level_down': False, # whether to look in ALL subfolders when searching for tiffs 'data_path': [], # a list of folders with tiffs # (or folder of folders with tiffs if look_one_level_down is True, or subfolders is not empty) 'subfolders': [] # choose subfolders of 'data_path' to look in (optional) } opsEnd=run_s2p(ops=ops,db=db) stat = np.load("suite2p/plane0/stat.npy") F = np.load("suite2p/plane0/F.npy") Fneu = np.load("suite2p/plane0/Fneu.npy") spatial_components = np.zeros((video.shape[2], video.shape[3], len(stat))) for i in range(len(stat)): spatial_components[stat[i]['xpix'], stat[i]['ypix'], i] = stat[i]['lam'] roi_spatial_footprints[z] = scipy.sparse.coo_matrix(spatial_components.reshape((video.shape[2]*video.shape[3], len(stat)))) roi_temporal_footprints[z] = F - ops["neucoeff"]*Fneu roi_temporal_residuals[z] = np.zeros(F.shape) bg_spatial_footprints[z] = None bg_temporal_footprints[z] = None os.remove(fname) shutil.rmtree("suite2p") return roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints
def run_suite2p(path_hdf5, diameter): save_path = os.path.dirname(path_hdf5) ops['diameter'] = diameter db['h5py'] = path_hdf5 db['save_path0'] = save_path run_s2p(ops=ops, db=db)
def main(): # set your options for running # overwrites the run_s2p.default_ops ops = { 'fast_disk': [], # used to store temporary binary file, defaults to save_path0 (set as a string NOT a list) 'save_path0': [], # stores results, defaults to first item in data_path 'delete_bin': False, # whether to delete binary file after processing # main settings 'nplanes': 1, # each tiff has these many planes in sequence 'nchannels': 1, # each tiff has these many channels per plane 'functional_chan': 1, # this channel is used to extract functional ROIs (1-based) 'diameter': 3, # this is the main parameter for cell detection, 2-dimensional if Y and X are different (e.g. [6 12]) 'tau': 2., # this is the main parameter for deconvolution, 2 for gcamp6s 'fs': 10., # sampling rate (total across planes) # output settings 'save_mat': False, # whether to save output as matlab files 'combined': True, # combine multiple planes into a single result /single canvas for GUI # parallel settings 'num_workers': 0, # 0 to select num_cores, -1 to disable parallelism, N to enforce value 'num_workers_roi': -1, # 0 to select number of planes, -1 to disable parallelism, N to enforce value # registration settings 'do_registration': False, # whether to register data 'nimg_init': 200, # subsampled frames for finding reference image 'batch_size': 200, # number of frames per batch 'maxregshift': 0.1, # max allowed registration shift, as a fraction of frame max(width and height) 'align_by_chan': 1, # when multi-channel, you can align by non-functional channel (1-based) 'reg_tif': False, # whether to save registered tiffs 'subpixel': 10, # precision of subpixel registration (1/subpixel steps) # cell detection settings 'connected': True, # whether or not to keep ROIs fully connected (set to 0 for dendrites) 'navg_frames_svd': 5000, # max number of binned frames for the SVD 'nsvd_for_roi': 1000, # max number of SVD components to keep for ROI detection 'max_iterations': 100, # maximum number of iterations to do cell detection 'ratio_neuropil': 6., # ratio between neuropil basis size and cell radius 'ratio_neuropil_to_cell': 3, # minimum ratio between neuropil radius and cell radius 'tile_factor': 1., # use finer (>1) or coarser (<1) tiles for neuropil estimation during cell detection # TODO: Try to lower the threshold to get more ROI 'threshold_scaling': 0.7, # adjust the automatically determined threshold by this scalar multiplier 'max_overlap': 0.70, # cells with more overlap than this get removed during triage, before refinement 'inner_neuropil_radius': 2, # number of pixels to keep between ROI and neuropil donut 'outer_neuropil_radius': np.inf, # maximum neuropil radius 'min_neuropil_pixels': 300, # minimum number of pixels in the neuropil # deconvolution settings 'baseline': 'maximin', # baselining mode 'win_baseline': 60., # window for maximin 'sig_baseline': 10., # smoothing constant for gaussian filter 'prctile_baseline': 8., # optional (whether to use a percentile baseline) 'neucoeff': .7, # neuropil coefficient } for_axon_flex = True if for_axon_flex: # new OPS (cop/past from https://github.com/MouseLand/suite2p on the 1st december 2019 ops = { # file paths 'look_one_level_down': False, # whether to look in all subfolders when searching for tiffs 'fast_disk': [], # used to store temporary binary file, defaults to save_path0 'delete_bin': False, # whether to delete binary file after processing 'mesoscan': False, # for reading in scanimage mesoscope files 'h5py': [], # take h5py as input (deactivates data_path) 'h5py_key': 'data', # key in h5py where data array is stored 'save_path0': [], # stores results, defaults to first item in data_path 'subfolders': [], # main settings 'nplanes': 1, # each tiff has these many planes in sequence 'nchannels': 1, # each tiff has these many channels per plane 'functional_chan': 1, # this channel is used to extract functional ROIs (1-based) 'tau': 2., # this is the main parameter for deconvolution, 2 for gcamp6s 'fs': 10., # sampling rate (PER PLANE - e.g. if you have 12 planes then this should be around 2.5) 'force_sktiff': False, # whether or not to use scikit-image for tiff reading # output settings 'preclassify': 0, # apply classifier before signal extraction with probability 0.5 (turn off with value 0) 'save_mat': False, # whether to save output as matlab files 'combined': True, # combine multiple planes into a single result /single canvas for GUI 'aspect': 1.0, # um/pixels in X / um/pixels in Y (for correct aspect ratio in GUI) # bidirectional phase offset 'do_bidiphase': False, 'bidiphase': 0, # registration settings 'do_registration': 0, # whether to register data (2 forces re-registration) 'keep_movie_raw': False, 'nimg_init': 300, # subsampled frames for finding reference image 'batch_size': 500, # number of frames per batch 'maxregshift': 0.1, # max allowed registration shift, as a fraction of frame max(width and height) 'align_by_chan': 1, # when multi-channel, you can align by non-functional channel (1-based) 'reg_tif': False, # whether to save registered tiffs 'reg_tif_chan2': False, # whether to save channel 2 registered tiffs 'subpixel': 10, # precision of subpixel registration (1/subpixel steps) 'smooth_sigma': 1.15, # ~1 good for 2P recordings, recommend >5 for 1P recordings 'th_badframes': 1.0, # this parameter determines which frames to exclude when determining cropping - set it smaller to exclude more frames 'pad_fft': False, # non rigid registration settings 'nonrigid': True, # whether to use nonrigid registration 'block_size': [ 128, 128 ], # block size to register (** keep this a multiple of 2 **) 'snr_thresh': 1.2, # if any nonrigid block is below this threshold, it gets smoothed until above this threshold. 1.0 results in no smoothing 'maxregshiftNR': 5, # maximum pixel shift allowed for nonrigid, relative to rigid # 1P settings '1Preg': False, # whether to perform high-pass filtering and tapering 'spatial_hp': 50, # window for spatial high-pass filtering before registration 'pre_smooth': 2, # whether to smooth before high-pass filtering before registration 'spatial_taper': 50, # how much to ignore on edges (important for vignetted windows, for FFT padding do not set BELOW 3*ops['smooth_sigma']) # cell detection settings 'roidetect': True, # whether or not to run ROI extraction 'spatial_scale': 0, # 0: multi-scale; 1: 6 pixels, 2: 12 pixels, 3: 24 pixels, 4: 48 pixels 'connected': 0, # whether or not to keep ROIs fully connected (set to 0 for dendrites) 'nbinned': 5000, # max number of binned frames for cell detection 'max_iterations': 100, # maximum number of iterations to do cell detection 'threshold_scaling': 1, # adjust the automatically determined threshold by this scalar multiplier used to be 5 'max_overlap': 0.7, # cells with more overlap than this get removed during triage, before refinement default: 0.75 'high_pass': 100, # running mean subtraction with window of size 'high_pass' (use low values for 1P) # ROI extraction parameters 'inner_neuropil_radius': 2, # number of pixels to keep between ROI and neuropil donut 'min_neuropil_pixels': 300, # minimum number of pixels in the neuropil 'allow_overlap': True, # pixels that are overlapping are thrown out (False) or added to both ROIs (True) # channel 2 detection settings (stat[n]['chan2'], stat[n]['not_chan2']) 'chan2_thres': 0.65, # minimum for detection of brightness on channel 2 # deconvolution settings 'baseline': 'maximin', # baselining mode (can also choose 'prctile') 'win_baseline': 60., # window for maximin 'sig_baseline': 10., # smoothing constant for gaussian filter 'prctile_baseline': 8., # optional (whether to use a percentile baseline) 'neucoeff': .7, # neuropil coefficient 'xrange': np.array([0, 0]), 'yrange': np.array([0, 0]) } else: # new OPS (cop/past from https://github.com/MouseLand/suite2p on the 1st december 2019 ops = { # file paths 'look_one_level_down': False, # whether to look in all subfolders when searching for tiffs 'fast_disk': [], # used to store temporary binary file, defaults to save_path0 'delete_bin': False, # whether to delete binary file after processing 'mesoscan': False, # for reading in scanimage mesoscope files 'h5py': [], # take h5py as input (deactivates data_path) 'h5py_key': 'data', # key in h5py where data array is stored 'save_path0': [], # stores results, defaults to first item in data_path 'subfolders': [], # main settings 'nplanes': 1, # each tiff has these many planes in sequence 'nchannels': 1, # each tiff has these many channels per plane 'functional_chan': 1, # this channel is used to extract functional ROIs (1-based) 'tau': 2., # this is the main parameter for deconvolution, 2 for gcamp6s 'fs': 10., # sampling rate (PER PLANE - e.g. if you have 12 planes then this should be around 2.5) 'force_sktiff': False, # whether or not to use scikit-image for tiff reading # output settings 'preclassify': 0, # apply classifier before signal extraction with probability 0.5 (turn off with value 0) 'save_mat': False, # whether to save output as matlab files 'combined': True, # combine multiple planes into a single result /single canvas for GUI 'aspect': 1.0, # um/pixels in X / um/pixels in Y (for correct aspect ratio in GUI) # bidirectional phase offset 'do_bidiphase': False, 'bidiphase': 0, # registration settings 'do_registration': 0, # whether to register data (2 forces re-registration) 'keep_movie_raw': False, 'nimg_init': 300, # subsampled frames for finding reference image 'batch_size': 500, # number of frames per batch 'maxregshift': 0.1, # max allowed registration shift, as a fraction of frame max(width and height) 'align_by_chan': 1, # when multi-channel, you can align by non-functional channel (1-based) 'reg_tif': False, # whether to save registered tiffs 'reg_tif_chan2': False, # whether to save channel 2 registered tiffs 'subpixel': 10, # precision of subpixel registration (1/subpixel steps) 'smooth_sigma': 1.15, # ~1 good for 2P recordings, recommend >5 for 1P recordings 'th_badframes': 1.0, # this parameter determines which frames to exclude when determining cropping - set it smaller to exclude more frames 'pad_fft': False, # non rigid registration settings 'nonrigid': True, # whether to use nonrigid registration 'block_size': [ 128, 128 ], # block size to register (** keep this a multiple of 2 **) 'snr_thresh': 1.2, # if any nonrigid block is below this threshold, it gets smoothed until above this threshold. 1.0 results in no smoothing 'maxregshiftNR': 5, # maximum pixel shift allowed for nonrigid, relative to rigid # 1P settings '1Preg': False, # whether to perform high-pass filtering and tapering 'spatial_hp': 50, # window for spatial high-pass filtering before registration 'pre_smooth': 2, # whether to smooth before high-pass filtering before registration 'spatial_taper': 50, # how much to ignore on edges (important for vignetted windows, for FFT padding do not set BELOW 3*ops['smooth_sigma']) # cell detection settings 'roidetect': True, # whether or not to run ROI extraction 'spatial_scale': 0, # 0: multi-scale; 1: 6 pixels, 2: 12 pixels, 3: 24 pixels, 4: 48 pixels 'connected': True, # whether or not to keep ROIs fully connected (set to 0 for dendrites) 'nbinned': 5000, # max number of binned frames for cell detection 'max_iterations': 100, # maximum number of iterations to do cell detection 'threshold_scaling': 1, # adjust the automatically determined threshold by this scalar multiplier used to be 5 'max_overlap': 0.7, # cells with more overlap than this get removed during triage, before refinement default: 0.75 'high_pass': 100, # running mean subtraction with window of size 'high_pass' (use low values for 1P) # ROI extraction parameters 'inner_neuropil_radius': 2, # number of pixels to keep between ROI and neuropil donut 'min_neuropil_pixels': 300, # minimum number of pixels in the neuropil 'allow_overlap': True, # pixels that are overlapping are thrown out (False) or added to both ROIs (True) # channel 2 detection settings (stat[n]['chan2'], stat[n]['not_chan2']) 'chan2_thres': 0.65, # minimum for detection of brightness on channel 2 # deconvolution settings 'baseline': 'maximin', # baselining mode (can also choose 'prctile') 'win_baseline': 60., # window for maximin 'sig_baseline': 10., # smoothing constant for gaussian filter 'prctile_baseline': 8., # optional (whether to use a percentile baseline) 'neucoeff': .7, # neuropil coefficient 'xrange': np.array([0, 0]), 'yrange': np.array([0, 0]) } """ """ """ ops to modify defaults): 'preclassify': 0.5 'threshold_scaling': 5 # select many more cells if lowered 'max_overlap': 0.75 'inner_neuropil_radius': 2, # number of pixels to keep between ROI and neuropil donut 'min_neuropil_pixels': 350, # minimum number of pixels in the neuropil 'allow_overlap': False, # pixels that are overlapping are thrown out (False) or added to both ROIs (True) 'neucoeff': .7, # neuropil coefficient 'allow_overlap' To run the code: conda activate suite2p (First time only: To install it from sources: pip install -e .) (after pasting the .h5 file in suite2p_main.py) then: python suite2p_main.py And to open the GUI: python -m suite2p At the end: conda deactivate """ # provide an h5 path in 'h5py' or a tiff path in 'data_path' # db overwrites any ops (allows for experiment specific settings) # p8_19_09_29_1_a001.h5 db = { # 'h5py': '/Users/pappyhammer/Documents/academique/these_inmed/suite2p/suite2p_tiffs/MichelMotC_p8_19_09_29_1_a000.h5', # a single h5 file path # 'h5py': '/home/julien/these_inmed/suite2p/suite2p_tiffs/181016_181029_a002_concat.h5', # 'h5py_key': 'data', 'look_one_level_down': False, # whether to look in ALL subfolders when searching for tiffs 'data_path': ['/home/julien/these_inmed/suite2p/suite2p_tiffs' ], # a list of folders with tiffs # 'data_path': ['/Users/pappyhammer/Documents/academique/these_inmed/suite2p/suite2p_tiffs'], # (or folder of folders with tiffs if look_one_level_down is True, or subfolders is not empty) 'subfolders': [], # choose subfolders of 'data_path' to look in (optional) 'fast_disk': '/home/julien/these_inmed/suite2p/suite2p_bin', # string which specifies where the binary file will be stored (should be an SSD) # 'fast_disk': '/Users/pappyhammer/Documents/academique/these_inmed/suite2p/suite2p_bin' } # run one experiment opsEnd = run_s2p(ops=ops, db=db)
def main(mouse_id, run_number, pkl_path, do_s2p=False, reprocess=True, reload=True, do_flu_preprocess=True): ''' caches an single session for one mouse to pkl''' run = run_processor(mouse_id, run_number, pkl_path, reprocess=reprocess, reload=reload) if run is None: return # flatten out the tseries path lists if it is nested if any(isinstance(i, list) for i in run.tseries_paths): run.tseries_paths = [ item for sublist in run.tseries_paths for item in sublist ] tseries_nframes = [] tseries_dims = [] print('\nfollowing tseries found:') tiff_list = [] print(run.tseries_paths) for tseries in run.tseries_paths: tiffs = utils.get_tiffs(tseries) if not tiffs: raise filenotfounderror('cannot find tiff in ' 'folder {}'.format(tseries)) elif len(tiffs) == 1: assert tiffs[0][-7:] == 'Ch3.tif', 'channel not understood '\ 'for tiff {}'.format(tiff) tiff_list.append(tiffs[0]) elif len(tiffs) == 2: # two channels recorded (red is too dim) assert tiffs[0][-7:] == 'Ch2.tif' and tiffs[1][-7:] == 'Ch3.tif',\ 'channel not understood '\ 'for tiffs {} and {}'.format(tiffs[0], tiffs[1]) tiffs = [tiffs[1]] tiff_list.append(tiffs[0]) elif len(tiffs) > 2: print('Single frame tiffs not deleted, checking for MPTs') # Check if multipage tiff conversion happend but single tiffs # were not deleted multipages = [tiff for tiff in tiffs if not '.ome' in tiff] multipages.sort() if not multipages: raise ValueError('Folder likely contains unconverted single ' 'page tiffs') # check that the number of tiffs in the multipage matches the # number of .ome num_ome = len([tiff for tiff in tiffs if '.ome' in tiff]) for i, multipage in enumerate(multipages): _, n_frames = tiff_metadata(multipage) # ideally want equal but sometimes deletion starts but is # not completed assert n_frames >= num_ome or n_frames == num_ome/len(multipages), \ '{} {} {}'.format(multipage, n_frames, num_ome) # check for the green channel if len(multipages) > 1: assert multipages[1][-7:] == 'Ch3.tif', 'channel not understood '\ 'for tiff {}'.format(multipages) tiff_list.append(multipages[1]) tiffs = [multipages[1]] elif len(multipages) == 1: assert multipages[0][-7:] == 'Ch3.tif', 'channel not understood '\ 'for tiff {}'.format(multipages) tiff_list.append(multipages[0]) tiffs = [multipages[0]] image_dims, n_frames = tiff_metadata(tiffs[0]) tseries_dims.append(image_dims) tseries_nframes.append(n_frames) print('\n') run.num_frames = tseries_nframes print(tseries_dims) if tseries_dims[0][0] == 1024 and tseries_dims[0][1] == 1024: diameter = 11 fs = 15 print('Bafov detected') raise # path hack in db has ruined bafov elif tseries_dims[0][0] == 1024 and tseries_dims[0][1] == 514: diameter = 11 fs = 30 assert run.num_planes == 1 # temporary print('obfov detected') else: raise NotImplementedError('Cacher is currently only set up for bafov ' 'and obfov images') data_path = str(Path(run.tseries_paths[0]).parent) save_folder = os.path.join(data_path, 'suite2p', run.mouse_id) if not os.path.exists(save_folder): os.makedirs(save_folder) db = { 'data_path': [data_path], 'look_one_level_down': True, 'diameter': diameter, 'tiff_list': tiff_list, 'nplanes': run.num_planes, 'fs': fs, 'save_folder': save_folder, 'roidetect': True, 'spikedetect': False, } # TEMPORARRY?? run.s2p_path = os.path.join(Path(db['data_path'][0]), 'suite2p') # 2021 addition, think this is correct run.s2p_path = save_folder # # check that suite2p hasn't alreay been run # if os.path.exists(run.s2p_path): # print('Already done s2p\n') # do_s2p = False print('Data path is {}'.format(data_path)) print('s2p path is {}'.format(run.s2p_path)) if do_s2p: # Build bad_frames.npy (not sure which directory to use so do both # useful to keep record of bad frames in mouse specific suite2p # output folder rf.get_bad_frames(run, [data_path, run.s2p_path, save_folder]) print('Running s2p on tseries printed above\n') with open('/home/jrowland/mnt/qnap/suite2p_report.txt', 'a') as f: dateTime = datetime.datetime.today() f.write('\n{} Beginning s2p for {} run {} path {}'.format( dateTime.isoformat("|", 'seconds'), run.mouse_id, run_number, save_folder)) opsEnd = run_s2p(ops=ops, db=db) with open('/home/jrowland/mnt/qnap/suite2p_report.txt', 'a') as f: dateTime = datetime.datetime.today() f.write('\n{} Completed s2p for {} run {} path {}\n\n'.format( dateTime.isoformat("|", 'seconds'), run.mouse_id, run_number, save_folder)) if do_flu_preprocess: run = preprocess_flu(run) if not run.reloaded: with open(pkl_file, 'wb') as f: pickle.dump(run, f)