def sorting_sync_and_alf(session_path, overwrite=False):
"""
Multiple steps. For each probe:
- Runs ks2 (skips if it already ran)
- synchronize the spike sorting
- output the probe description files
:param overwrite:
:return: list of files to be registered on database
"""
efiles = spikeglx.glob_ephys_files(session_path)
ap_files = [(ef.get("ap"), ef.get("label")) for ef in efiles
if "ap" in ef.keys()]
out_files = []
for ap_file, label in ap_files:
ks2_dir = session_path.joinpath("spike_sorters", "ks2_matlab", label)
probe_out_path = session_path.joinpath("alf", label)
probe_out_path.mkdir(parents=True, exist_ok=True)
spikes.ks2_to_alf(
ks2_dir,
bin_path=ap_file.parent,
out_path=probe_out_path,
bin_file=ap_file,
ampfactor=_sample2v(ap_file),
)
out, _ = spikes.sync_spike_sorting(ap_file=ap_file,
out_path=probe_out_path)
out_files.extend(out)
# convert ks2_output into tar file and also register
# Make this in case spike sorting is in old raw_ephys_data folders, for new
# sessions it should already exist
tar_dir = session_path.joinpath('spike_sorters', 'ks2_matlab', label)
tar_dir.mkdir(parents=True, exist_ok=True)
out = spikes.ks2_to_tar(ks2_dir, tar_dir)
out_files.extend(out)
def upload_ks2_output():
"""
Copy ks2 output to a .tar file and upload to flatiron for all past sessions that have
spike sorting output
"""
# if the space on the disk > 500Gb continue, otherwise, don't bother
usage = _get_volume_usage('/mnt/s0/Data', 'disk')
if usage['disk_available'] < 500:
return
one = ONE()
for ilog, ks2_out in enumerate(ROOT_PATH.rglob('spike_sorting_ks2.log')):
# check space on disk after every 25 extractions. stop if we are running low!
if np.mod(ilog, 25) == 0:
usage = _get_volume_usage('/mnt/s0/Data', 'disk')
if usage['disk_available'] < 500:
return
ks2_path = Path(ks2_out).parent
session_path = alf.io.get_session_path(ks2_out)
probe = ks2_path.stem
tar_dir = session_path.joinpath('spike_sorters', 'ks2_matlab', probe)
tar_dir.mkdir(exist_ok=True, parents=True)
# If the flag exists it means we have already extracted
if tar_dir.joinpath('tar_existed.flag').exists():
# We already done this, no need to repeat!!
continue
eid = one.eid_from_path(session_path)
# For latest sessions tar file will be created by task and automatically registered so we
# may have a case where tar file already registered and uploaded but no tar_existed.flag
if tar_dir.joinpath('_kilosort_raw.output.tar').exists():
# double check it indeed has been registered for this probe
dset = one.alyx.rest('datasets',
'list',
session=eid,
name='_kilosort_raw.output.tar')
collection = [ds['collection'].rsplit('/', 1)[-1] for ds in dset]
if probe in collection:
tar_dir.joinpath('tar_existed.flag').touch()
continue
if eid is None:
# Skip sessions that don't exist on alyx!
continue
out = spikes.ks2_to_tar(ks2_path, tar_dir)
register_dataset(out, one=one)
# Make flag to indicate data already registered for this session
tar_dir.joinpath('tar_existed.flag').touch()
def _run(self, overwrite=False):
"""
Multiple steps. For each probe:
- Runs ks2 (skips if it already ran)
- synchronize the spike sorting
- output the probe description files
:param overwrite:
:return: list of files to be registered on database
"""
efiles = spikeglx.glob_ephys_files(self.session_path)
ap_files = [(ef.get("ap"), ef.get("label")) for ef in efiles
if "ap" in ef.keys()]
out_files = []
for ap_file, label in ap_files:
try:
ks2_dir = self._run_ks2(
ap_file) # runs ks2, skips if it already ran
probe_out_path = self.session_path.joinpath("alf", label)
probe_out_path.mkdir(parents=True, exist_ok=True)
spikes.ks2_to_alf(
ks2_dir,
bin_path=ap_file.parent,
out_path=probe_out_path,
bin_file=ap_file,
ampfactor=self._sample2v(ap_file),
)
out, _ = spikes.sync_spike_sorting(ap_file=ap_file,
out_path=probe_out_path)
out_files.extend(out)
# convert ks2_output into tar file and also register
# Make this in case spike sorting is in old raw_ephys_data folders, for new
# sessions it should already exist
tar_dir = self.session_path.joinpath('spike_sorters',
'ks2_matlab', label)
tar_dir.mkdir(parents=True, exist_ok=True)
out = spikes.ks2_to_tar(ks2_dir, tar_dir)
out_files.extend(out)
except BaseException:
_logger.error(traceback.format_exc())
self.status = -1
continue
probe_files = spikes.probes_description(self.session_path,
one=self.one)
return out_files + probe_files