def sync_merge_ephys(root_data_folder, dry=False): """ Post spike-sorting processing: - synchronization of probes - ks2 to ALF conversion for each probes in alf/probeXX folder - computes spike sorting QC - creates probes object in alf folder To start the job for a session, all electrophysiology ap files from session need to be associated with a `sync_merge_ephys.flag` file Outputs individual probes """ syncflags = list(Path(root_data_folder).rglob('sync_merge_ephys.flag')) session_paths = list(set([f.parents[2] for f in syncflags])) for session_path in session_paths: print(session_path) if dry: continue # first remove the flags [f.unlink() for f in syncflags if f.parents[2] == session_path] # first sync the probes sync_probes.sync(session_path) # then convert ks2 to ALF and resync spike sorting data spikes.sync_spike_sortings(session_path) # outputs the probes object in the ALF folder spikes.probes_description(session_path) # wrap up by removing flags and creating register_me flag flags.write_flag_file(session_path.joinpath('register_me.flag'))
def extract_pulses(session_path, overwrite=False): # outputs numpy syncs, out_files = ephys_fpga.extract_sync(session_path, overwrite=overwrite) for out_file in out_files: _logger.info(f"extracted pulses for {out_file}") status, sync_files = sync_probes.sync(session_path) return out_files + sync_files
def sync_spike_sorting(ap_file, out_path): """ Synchronizes the spike.times using the previously computed sync files :param ap_file: raw binary data file for the probe insertion :param out_path: probe output path (usually {session_path}/alf/{probe_label}) """ def _sr(ap_file): # gets sampling rate from data md = spikeglx.read_meta_data(ap_file.with_suffix('.meta')) return spikeglx._get_fs_from_meta(md) out_files = [] label = ap_file.parts[ -1] # now the bin file is always in a folder bearing the name of probe sync_file = ap_file.parent.joinpath(ap_file.name.replace( '.ap.', '.sync.')).with_suffix('.npy') # try to get probe sync if it doesn't exist if not sync_file.exists(): _, sync_files = sync_probes.sync(get_session_path(ap_file)) out_files.extend(sync_files) # if it still not there, full blown error if not sync_file.exists(): # if there is no sync file it means something went wrong. Outputs the spike sorting # in time according the the probe by following ALF convention on the times objects error_msg = f'No synchronisation file for {label}: {sync_file}. The spike-' \ f'sorting is not synchronized and data not uploaded on Flat-Iron' _logger.error(error_msg) # remove the alf folder if the sync failed shutil.rmtree(out_path) return None, 1 # patch the spikes.times files manually st_file = out_path.joinpath('spikes.times.npy') spike_samples = np.load(out_path.joinpath('spikes.samples.npy')) interp_times = apply_sync(sync_file, spike_samples / _sr(ap_file), forward=True) np.save(st_file, interp_times) # get the list of output files out_files.extend([ f for f in out_path.glob("*.*") if f.name.startswith(('channels.', 'drift', 'clusters.', 'spikes.', 'templates.', '_kilosort_', '_phy_spikes_subset', '_ibl_log.info')) ]) # the QC files computed during spike sorting stay within the raw ephys data folder out_files.extend(list(ap_file.parent.glob('_iblqc_*AP.*.npy'))) return out_files, 0
def _run(self): """ Post spike-sorting processing: - synchronization of probes - ks2 to ALF conversion for each probes in alf/probeXX folder - computes spike sorting QC - creates probes object in alf folder To start the job for a session, all electrophysiology ap files from session need to be associated with a `sync_merge_ephys.flag` file Outputs individual probes """ # first sync the probes status, sync_files = sync_probes.sync(self.session_path) # then convert ks2 to ALF and resync spike sorting data alf_files = spikes.sync_spike_sortings(self.session_path) # outputs the probes object in the ALF folder probe_files = spikes.probes_description(self.session_path) return sync_files + alf_files + probe_files
def job_run_ks2(): # Look for flag files in /mnt/s0/Data and sort them in order of date they were created flag_files = list(Path(root_path).glob('**/sort_me.flag')) flag_files.sort(key=os.path.getmtime) # Start with the oldest flag session_path = flag_files[0].parent session = str(PosixPath(*session_path.parts[4:])) flag_files[0].unlink() # Instantiate one one = ONE(cache_rest=None) # sync the probes status, sync_files = sync_probes.sync(session_path) if not status: _logger.error(f'{session}: Could not sync probes') return else: _logger.info(f'{session}: Probes successfully synced') # run ks2 task = ephys.SpikeSorting(session_path, one=one) status = task.run() if status != 0: _logger.error(f'{session}: Could not run ks2') return else: _logger.info(f'{session}: ks2 successfully completed') # Run the cell qc # qc_file = [] # Register and upload files to FTP Patcher outfiles = task.outputs ftp_patcher = FTPPatcher(one=one) ftp_patcher.create_dataset(path=outfiles, created_by=one._par.ALYX_LOGIN)