def test_glob_ephys(self):
def dict_equals(d1, d2):
return all([x in d1 for x in d2]) and all([x in d2 for x in d1])
ef3b = spikeglx.glob_ephys_files(self.dir3b)
ef3a = spikeglx.glob_ephys_files(self.dir3a)
ef3b_ch = spikeglx.glob_ephys_files(self.dir3b, ext='ch')
# test glob
self.assertTrue(dict_equals(self.dict3a, ef3a))
self.assertTrue(dict_equals(self.dict3b, ef3b))
self.assertTrue(dict_equals(self.dict3b_ch, ef3b_ch))
# test the version from glob
self.assertTrue(
spikeglx.get_neuropixel_version_from_files(ef3a) == '3A')
self.assertTrue(
spikeglx.get_neuropixel_version_from_files(ef3b) == '3B')
# test the version from paths
self.assertTrue(
spikeglx.get_neuropixel_version_from_folder(self.dir3a) == '3A')
self.assertTrue(
spikeglx.get_neuropixel_version_from_folder(self.dir3b) == '3B')
self.dir3b.joinpath('imec1',
'sync_testing_g0_t0.imec1.ap.bin').unlink()
self.assertEqual(
spikeglx.glob_ephys_files(self.dir3b.joinpath('imec1')), [])
def _run(self):
"""
Compress ephys files looking for `compress_ephys.flag` whithin the probes folder
Original bin file will be removed
The registration flag created contains targeted file names at the root of the session
"""
out_files = []
ephys_files = spikeglx.glob_ephys_files(self.session_path)
ephys_files += spikeglx.glob_ephys_files(self.session_path, ext="ch")
ephys_files += spikeglx.glob_ephys_files(self.session_path, ext="meta")
for ef in ephys_files:
for typ in ["ap", "lf", "nidq"]:
bin_file = ef.get(typ)
if not bin_file:
continue
if bin_file.suffix.find("bin") == 1:
sr = spikeglx.Reader(bin_file)
if sr.is_mtscomp:
out_files.append(bin_file)
else:
_logger.info(f"Compressing binary file {bin_file}")
out_files.append(sr.compress_file(keep_original=False))
out_files.append(bin_file.with_suffix('.ch'))
else:
out_files.append(bin_file)
return out_files
def test_get_ephys_files(self):
# first test at the root directory level, with a string input
ephys_files = spikeglx.glob_ephys_files(self.dir.name)
for ef in ephys_files:
self.assertTrue(ef.label in ['probe_right', 'probe_left'])
self.assertTrue(ef.ap.exists() and ef.lf.exists())
# second test at the ephys directory level, with a pathlib.Path input
ephys_files = spikeglx.glob_ephys_files(
Path(self.dir.name) / 'raw_ephys_data')
for ef in ephys_files:
self.assertTrue(ef.label in ['probe_right', 'probe_left'])
self.assertTrue(ef.ap.exists() and ef.lf.exists())
def compress_ephys(root_data_folder, dry=False, max_sessions=5):
"""
Compress ephys files looking for `compress_ephys.flag` whithin the probes folder
Original bin file will be removed
The registration flag created contains targeted file names at the root of the session
"""
qcflags = Path(root_data_folder).rglob('compress_ephys.flag')
c = 0
for qcflag in qcflags:
probe_path = qcflag.parent
c += 1
if c > max_sessions:
return
if dry:
print(qcflag.parent)
continue
# no rglob: only the folder in which the flag is located gets searched
ephys_files = spikeglx.glob_ephys_files(probe_path, recursive=False)
out_files = []
for ef in ephys_files:
for typ in ['ap', 'lf', 'nidq']:
bin_file = ef.get(typ)
if not bin_file:
continue
sr = spikeglx.Reader(bin_file)
if not sr.is_mtscomp:
out_files.append(sr.compress_file(keep_original=False))
qcflag.unlink()
if out_files:
session_path = probe_path.parents[1]
file_list = [str(f.relative_to(session_path)) for f in out_files]
flags.write_flag_file(probe_path.joinpath('register_me.flag'), file_list=file_list)
def extract_sync(session_path, save=False, force=False, ephys_files=None):
"""
Reads ephys binary file (s) and extract sync within the binary file folder
Assumes ephys data is within a `raw_ephys_data` folder
:param session_path: '/path/to/subject/yyyy-mm-dd/001'
:param save: Bool, defaults to False
:param force: Bool on re-extraction, forces overwrite instead of loading existing sync files
:return: list of sync dictionaries
"""
session_path = Path(session_path)
if not ephys_files:
ephys_files = glob_ephys_files(session_path)
syncs = []
for efi in ephys_files:
glob_filter = f'*{efi.label}*' if efi.label else '*'
bin_file = efi.get('ap', efi.get('nidq', None))
if not bin_file:
continue
file_exists = alf.io.exists(bin_file.parent, object='_spikeglx_sync', glob=glob_filter)
if not force and file_exists:
_logger.warning(f'Skipping raw sync: SGLX sync found for probe {efi.label} !')
sync = alf.io.load_object(bin_file.parent, object='_spikeglx_sync', glob=glob_filter)
else:
sr = ibllib.io.spikeglx.Reader(bin_file)
sync = _sync_to_alf(sr, bin_file.parent, save=save, parts=efi.label)
syncs.extend([sync])
return syncs
def _check_session_sync(ses_path, channel):
"""
Resync the original cam pulses
:param ses_path:
:return:
"""
efiles = spikeglx.glob_ephys_files(ses_path, bin_exists=False)
tprobe = []
tinterp = []
for ef in efiles:
if not ef.get('ap'):
continue
sync_events = alf.io.load_object(ef.ap.parent, 'sync', short_keys=True)
# the first step is to construct list arrays with probe sync
sync_file = ef.ap.parent.joinpath(ef.ap.name.replace(
'.ap.', '.sync.')).with_suffix('.npy')
t = sync_events.times[sync_events.channels == channel]
tsync = sync_probes.apply_sync(sync_file, t, forward=True)
tprobe.append(t)
tinterp.append(tsync)
# the second step is to make sure sample / time_ref files match time / time_ref files
ts_file = ef.ap.parent.joinpath(
ef.ap.name.replace('.ap.', '.timestamps.')).with_suffix('.npy')
fs = spikeglx._get_fs_from_meta(
spikeglx.read_meta_data(ef.ap.with_suffix('.meta')))
tstamp = sync_probes.apply_sync(ts_file, t * fs, forward=True)
assert (np.all(tstamp - tsync < 1e-12))
return tinterp[0] - tinterp[1]
def _run(self, overwrite=False):
"""
Multiple steps. For each probe:
- Runs ks2 (skips if it already ran)
- synchronize the spike sorting
- output the probe description files
:param overwrite:
:return: list of files to be registered on database
"""
efiles = spikeglx.glob_ephys_files(self.session_path)
ap_files = [(ef.get('ap'), ef.get('label')) for ef in efiles
if 'ap' in ef.keys()]
out_files = []
for ap_file, label in ap_files:
try:
ks2_dir = self._run_ks2(
ap_file) # runs ks2, skips if it already ran
probe_out_path = self.session_path.joinpath('alf', label)
probe_out_path.mkdir(parents=True, exist_ok=True)
spikes.ks2_to_alf(ks2_dir,
bin_path=ap_file.parent,
out_path=probe_out_path,
bin_file=ap_file,
ampfactor=self._sample2v(ap_file))
out, _ = spikes.sync_spike_sorting(ap_file=ap_file,
out_path=probe_out_path)
out_files.extend(out)
except BaseException as err:
_logger.error(err)
self.status = -1
continue
probe_files = spikes.probes_description(self.session_path,
one=self.one)
return out_files + probe_files
def phy2alf_conversion(session_path, ks2_path, alf_path, probe_label):
try:
# Find spikeglx meta data files associated with the session and probe
files = spikeglx.glob_ephys_files(session_path, ext='meta')
ap_files = [(ef.get("ap"), ef.get("label")) for ef in files
if "ap" in ef.keys()]
meta_file = next(ap[0] for ap in ap_files if ap[1] == probe_label)
# The .cbin file doesn't always still exist on server so point to it from meta
ap_file = meta_file.with_suffix('.cbin')
# Convert to alf format
spikes.ks2_to_alf(
ks2_path,
bin_path=meta_file.parent,
out_path=alf_path,
bin_file=None,
ampfactor=SpikeSorting_KS2_Matlab._sample2v(ap_file))
# Sync the probes
out_files, _ = spikes.sync_spike_sorting(ap_file=ap_file,
out_path=alf_path)
return 0, out_files, None
except BaseException as err:
_logger.error(
f'{session_path} and {probe_label} errored with message: {err}'
)
return -1, None, err
def sorting_sync_and_alf(session_path, overwrite=False):
"""
Multiple steps. For each probe:
- Runs ks2 (skips if it already ran)
- synchronize the spike sorting
- output the probe description files
:param overwrite:
:return: list of files to be registered on database
"""
efiles = spikeglx.glob_ephys_files(session_path)
ap_files = [(ef.get("ap"), ef.get("label")) for ef in efiles
if "ap" in ef.keys()]
out_files = []
for ap_file, label in ap_files:
ks2_dir = session_path.joinpath("spike_sorters", "ks2_matlab", label)
probe_out_path = session_path.joinpath("alf", label)
probe_out_path.mkdir(parents=True, exist_ok=True)
spikes.ks2_to_alf(
ks2_dir,
bin_path=ap_file.parent,
out_path=probe_out_path,
bin_file=ap_file,
ampfactor=_sample2v(ap_file),
)
out, _ = spikes.sync_spike_sorting(ap_file=ap_file,
out_path=probe_out_path)
out_files.extend(out)
# convert ks2_output into tar file and also register
# Make this in case spike sorting is in old raw_ephys_data folders, for new
# sessions it should already exist
tar_dir = session_path.joinpath('spike_sorters', 'ks2_matlab', label)
tar_dir.mkdir(parents=True, exist_ok=True)
out = spikes.ks2_to_tar(ks2_dir, tar_dir)
out_files.extend(out)
def extract_sync(session_path, overwrite=False, ephys_files=None):
"""
Reads ephys binary file (s) and extract sync within the binary file folder
Assumes ephys data is within a `raw_ephys_data` folder
:param session_path: '/path/to/subject/yyyy-mm-dd/001'
:param overwrite: Bool on re-extraction, forces overwrite instead of loading existing files
:return: list of sync dictionaries
"""
session_path = Path(session_path)
if not ephys_files:
ephys_files = spikeglx.glob_ephys_files(session_path)
syncs = []
outputs = []
for efi in ephys_files:
bin_file = efi.get('ap', efi.get('nidq', None))
if not bin_file:
continue
alfname = dict(object='sync', namespace='spikeglx')
if efi.label:
alfname['extra'] = efi.label
file_exists = alfio.exists(bin_file.parent, **alfname)
if not overwrite and file_exists:
_logger.warning(f'Skipping raw sync: SGLX sync found for probe {efi.label} !')
sync = alfio.load_object(bin_file.parent, **alfname)
out_files, _ = alfio._ls(bin_file.parent, **alfname)
else:
sr = spikeglx.Reader(bin_file)
sync, out_files = _sync_to_alf(sr, bin_file.parent, save=True, parts=efi.label)
outputs.extend(out_files)
syncs.extend([sync])
return syncs, outputs
def raw_qc_session(session_path, dry=False, force=False):
"""
Wrapper that exectutes QC from a session folder and outputs the results whithin the same folder
as the original raw data.
:param session_path: path of the session (Subject/yyyy-mm-dd/number
:param dry: bool (False) Dry run if True
:param force: bool (False) Force means overwriting an existing QC file
:return: None
"""
efiles = spikeglx.glob_ephys_files(session_path)
for efile in efiles:
if dry:
print(efile.ap)
print(efile.lf)
continue
if efile.ap and efile.ap.exists():
extract_rmsmap(efile.ap,
out_folder=None,
force=force,
label=efile.label)
if efile.lf and efile.lf.exists():
extract_rmsmap(efile.lf,
out_folder=None,
force=force,
label=efile.label)
def sync_spike_sortings(ses_path):
"""
Converts the KS2 outputs for each probe in ALF format. Creates:
alf/probeXX/spikes.*
alf/probeXX/clusters.*
alf/probeXX/templates.*
:param ses_path: session containing probes to be merged
:return: None
"""
def _sr(ap_file):
# gets sampling rate from data
md = spikeglx.read_meta_data(ap_file.with_suffix('.meta'))
return spikeglx._get_fs_from_meta(md)
def _sample2v(ap_file):
md = spikeglx.read_meta_data(ap_file.with_suffix('.meta'))
s2v = spikeglx._conversion_sample2v_from_meta(md)
return s2v['ap'][0]
ses_path = Path(ses_path)
ephys_files = spikeglx.glob_ephys_files(ses_path)
subdirs, labels, efiles_sorted, srates = zip(
*sorted([(ep.ap.parent, ep.label, ep, _sr(ep.ap)) for ep in ephys_files if ep.get('ap')]))
_logger.info('converting spike-sorting outputs to ALF')
out_files = []
for subdir, label, ef, sr in zip(subdirs, labels, efiles_sorted, srates):
if not subdir.joinpath('spike_times.npy').exists():
_logger.warning(f"No KS2 spike sorting found in {subdir}, skipping probe !")
continue
probe_out_path = ses_path.joinpath('alf', label)
probe_out_path.mkdir(parents=True, exist_ok=True)
# handles the probes synchronization
sync_file = ef.ap.parent.joinpath(ef.ap.name.replace('.ap.', '.sync.')
).with_suffix('.npy')
if not sync_file.exists():
"""
if there is no sync file it means something went wrong. Outputs the spike sorting
in time according the the probe by following ALF convention on the times objects
"""
error_msg = f'No synchronisation file for {label}: {sync_file}. The spike-' \
f'sorting is not synchronized and data not uploaded on Flat-Iron'
_logger.error(error_msg)
# remove the alf folder if the sync failed
shutil.rmtree(probe_out_path)
continue
# converts the folder to ALF
ks2_to_alf(subdir, probe_out_path, ampfactor=_sample2v(ef.ap), label=None, force=True)
# patch the spikes.times files manually
st_file = ses_path.joinpath(probe_out_path, 'spikes.times.npy')
spike_samples = np.load(ses_path.joinpath(probe_out_path, 'spikes.samples.npy'))
interp_times = apply_sync(sync_file, spike_samples / sr, forward=True)
np.save(st_file, interp_times)
# get the list of output files
out_files.extend([f for f in ses_path.joinpath(probe_out_path).glob("*.*") if
f.name.startswith(('channels.', 'clusters.', 'spikes.', 'templates.',
'_kilosort_', '_phy_spikes_subset'))])
return out_files
def _get_all_probes_sync(session_path, bin_exists=True):
# round-up of all bin ephys files in the session, infer revision and get sync map
ephys_files = spikeglx.glob_ephys_files(session_path, bin_exists=bin_exists)
version = spikeglx.get_neuropixel_version_from_files(ephys_files)
# attach the sync information to each binary file found
for ef in ephys_files:
ef['sync'] = alfio.load_object(ef.path, 'sync', namespace='spikeglx', short_keys=True)
ef['sync_map'] = get_ibl_sync_map(ef, version)
return ephys_files
def test_compress_session(self):
EphysMtscomp(self.main_folder).run()
ephys_files = spikeglx.glob_ephys_files(self.main_folder)
for ef in ephys_files:
# there is only one compressed file afterwards
self.assertTrue(ef.ap.suffix == '.cbin')
self.assertFalse(ef.ap.with_suffix('.bin').exists())
# the compressed file is readable
sr = spikeglx.Reader(ef.ap)
self.assertTrue(sr.is_mtscomp)
def probes_description(ses_path, bin_exists=True):
"""
Aggregate probes information into ALF files
Input:
raw_ephys_data/probeXX/
Output:
alf/probes.description.npy
alf/probes.trajecory.npy
"""
ses_path = Path(ses_path)
ephys_files = spikeglx.glob_ephys_files(ses_path, bin_exists=bin_exists)
subdirs, labels, efiles_sorted = zip(
*sorted([(ep.ap.parent, ep.label, ep) for ep in ephys_files if ep.get('ap')]))
"""Ouputs the probes description file"""
probe_description = []
for label, ef in zip(labels, efiles_sorted):
md = spikeglx.read_meta_data(ef.ap.with_suffix('.meta'))
probe_description.append({'label': label,
'model': md.neuropixelVersion,
'serial': int(md.serial),
'raw_file_name': md.fileName,
})
alf_path = ses_path.joinpath('alf')
alf_path.mkdir(exist_ok=True, parents=True)
probe_description_file = alf_path.joinpath('probes.description.json')
with open(probe_description_file, 'w+') as fid:
fid.write(json.dumps(probe_description))
"""Ouputs the probes trajectory file"""
bpod_meta = raw_data_loaders.load_settings(ses_path)
if not bpod_meta.get('PROBE_DATA'):
_logger.error('No probe information in settings JSON. Skipping probes.trajectory')
return
def prb2alf(prb, label):
return {'label': label, 'x': prb['X'], 'y': prb['Y'], 'z': prb['Z'], 'phi': prb['A'],
'theta': prb['P'], 'depth': prb['D'], 'beta': prb['T']}
# the labels may not match, in which case throw a warning and work in alphabetical order
if labels != ('probe00', 'probe01'):
_logger.warning("Probe names do not match the json settings files. Will match coordinates"
" per alphabetical order !")
_ = [_logger.warning(f" probe0{i} ---------- {lab} ") for i, lab in enumerate(labels)]
trajs = []
keys = sorted(bpod_meta['PROBE_DATA'].keys())
for i, k in enumerate(keys):
if i >= len(labels):
break
trajs.append(prb2alf(bpod_meta['PROBE_DATA'][f'probe0{i}'], labels[i]))
probe_trajectory_file = alf_path.joinpath('probes.trajectory.json')
with open(probe_trajectory_file, 'w+') as fid:
fid.write(json.dumps(trajs))
return [probe_trajectory_file, probe_description_file]
def _run(self, overwrite=False):
efiles = spikeglx.glob_ephys_files(self.session_path)
apfiles = [ef.get('ap') for ef in efiles if 'ap' in ef.keys()]
for apfile in apfiles:
ks2log = apfile.parent.joinpath('spike_sorting_ks2.log')
if not ks2log.exists():
# this will label the job with "empty" status in the database
return None
with open(ks2log) as fid:
line = fid.readline()
self.version = re.compile("[a-f0-9]{36}").findall(line)[0]
return [] # the job will be labeled as complete with empty string
def _get_all_probes_sync(session_path):
# round-up of all bin ephys files in the session, infer revision and get sync map
ephys_files = glob_ephys_files(session_path)
version = get_neuropixel_version_from_files(ephys_files)
extract_sync(session_path, save=True)
# attach the sync information to each binary file found
for ef in ephys_files:
ef['sync'] = alf.io.load_object(ef.path,
'_spikeglx_sync',
short_keys=True)
ef['sync_map'] = get_ibl_sync_map(ef, version)
return ephys_files
def version3B(ses_path, display=True, type=None, tol=2.5):
"""
From a session path with _spikeglx_sync arrays extraccted, locate ephys files for 3A and
outputs one sync.timestamps.probeN.npy file per acquired probe. By convention the reference
probe is the one with the most synchronisation pulses.
Assumes the _spikeglx_sync datasets are already extracted from binary data
:param ses_path:
:param type: linear, exact or smooth
:return: None
"""
DEFAULT_TYPE = 'smooth'
ephys_files = spikeglx.glob_ephys_files(ses_path, bin_exists=False)
for ef in ephys_files:
ef['sync'] = alf.io.load_object(ef.path,
'sync',
namespace='spikeglx',
short_keys=True)
ef['sync_map'] = get_ibl_sync_map(ef, '3B')
nidq_file = [ef for ef in ephys_files if ef.get('nidq')]
ephys_files = [ef for ef in ephys_files if not ef.get('nidq')]
# should have at least 2 probes and only one nidq
assert (len(nidq_file) == 1)
nidq_file = nidq_file[0]
sync_nidq = _get_sync_fronts(nidq_file.sync,
nidq_file.sync_map['imec_sync'])
qc_all = True
out_files = []
for ef in ephys_files:
sync_probe = _get_sync_fronts(ef.sync, ef.sync_map['imec_sync'])
sr = _get_sr(ef)
try:
assert (sync_nidq.times.size == sync_probe.times.size)
except AssertionError:
raise Neuropixel3BSyncFrontsNonMatching(f"{ses_path}")
# if the qc of the diff finds anomalies, do not attempt to smooth the interp function
qcdiff = _check_diff_3b(sync_probe)
if not qcdiff:
qc_all = False
type_probe = type or 'exact'
else:
type_probe = type or DEFAULT_TYPE
timestamps, qc = sync_probe_front_times(sync_probe.times,
sync_nidq.times,
sr,
display=display,
type=type_probe,
tol=tol)
qc_all &= qc
out_files.extend(_save_timestamps_npy(ef, timestamps, sr))
return qc_all, out_files
def version3A(ses_path, display=True, linear=False, tol=1.5):
"""
From a session path with _spikeglx_sync arrays extracted, locate ephys files for 3A and
outputs one sync.timestamps.probeN.npy file per acquired probe. By convention the reference
probe is the one with the most synchronisation pulses.
Assumes the _spikeglx_sync datasets are already extracted from binary data
:param ses_path:
:return: bool True on a a successful sync
"""
ephys_files = spikeglx.glob_ephys_files(ses_path)
nprobes = len(ephys_files)
if nprobes <= 1:
_logger.warning(f"Skipping single probe session: {ses_path}")
return True
d = Bunch({'times': [], 'nsync': np.zeros(nprobes, )})
for ind, ephys_file in enumerate(ephys_files):
sync = alf.io.load_object(ephys_file.ap.parent,
'_spikeglx_sync',
short_keys=True)
sync_map = get_ibl_sync_map(ephys_file, '3A')
isync = np.in1d(sync['channels'], np.array([sync_map['right_camera']]))
d.nsync[ind] = len(sync.channels)
d['times'].append(sync['times'][isync])
# chop off to the lowest number of sync points
nsyncs = [t.size for t in d['times']]
if len(set(nsyncs)) > 1:
_logger.warning(
"Probes don't have the same number of synchronizations pulses")
d['times'] = np.r_[[t[:min(nsyncs)] for t in d['times']]].transpose()
# the reference probe is the one with the most sync pulses detected
iref = np.argmax(d.nsync)
# islave = np.setdiff1d(np.arange(nprobes), iref)
# get the sampling rate from the reference probe using metadata file
sr = _get_sr(ephys_files[iref])
qc_all = True
# output timestamps files as per ALF convention
for ind, ephys_file in enumerate(ephys_files):
if ind == iref:
timestamps = np.array([[0, 0], [1, 1]])
else:
timestamps, qc = sync_probe_front_times(d.times[:, iref],
d.times[:, ind],
sr,
display=display,
linear=linear,
tol=tol)
qc_all &= qc
_save_timestamps_npy(ephys_file, timestamps)
return qc_all
def extract_data(ks_path, ephys_path, out_path):
efiles = spikeglx.glob_ephys_files(ephys_path)
for efile in efiles:
if efile.get('ap') and efile.ap.exists():
ks2_to_alf(ks_path,
ephys_path,
out_path,
bin_file=efile.ap,
ampfactor=_sample2v(efile.ap),
label=None,
force=True)
extract_rmsmap(efile.ap, out_folder=out_path, spectra=False)
if efile.get('lf') and efile.lf.exists():
extract_rmsmap(efile.lf, out_folder=out_path)
def raw_qc_session(session_path, overwrite=False):
"""
Wrapper that exectutes QC from a session folder and outputs the results whithin the same folder
as the original raw data.
:param session_path: path of the session (Subject/yyyy-mm-dd/number
:param overwrite: bool (False) Force means overwriting an existing QC file
:return: None
"""
efiles = spikeglx.glob_ephys_files(session_path)
qc_files = []
for efile in efiles:
if efile.get('ap') and efile.ap.exists():
qc_files.extend(extract_rmsmap(efile.ap, out_folder=None, overwrite=overwrite))
if efile.get('lf') and efile.lf.exists():
qc_files.extend(extract_rmsmap(efile.lf, out_folder=None, overwrite=overwrite))
return qc_files
def check_ephys_file(root_path, hash=False):
root_path = Path(root_path)
efiles = spikeglx.glob_ephys_files(root_path)
for ef in efiles:
for lab in ['nidq', 'ap', 'lf']:
if not ef.get(lab, None):
continue
try:
sr = spikeglx.Reader(ef[lab])
if hash:
ok = sr.verify_hash()
if not ok:
raise ValueError("hashes don't match")
_logger.info(f"PASS {ef[lab]}")
except (Exception) as e:
_logger.error(f"FAILED {ef[lab]} is corrupt !!")
def load_lfp(eid, one=None, dataset_types=None):
"""
From an eid, hits the Alyx database and downloads the standard set of datasets
needed for LFP
:param eid:
:param dataset_types: additional dataset types to add to the list
:return: spikeglx.Reader
"""
if dataset_types is None:
dataset_types = []
dtypes = dataset_types + ['ephysData.raw.lf', 'ephysData.raw.meta', 'ephysData.raw.ch']
one.load(eid, dataset_types=dtypes, download_only=True)
session_path = one.path_from_eid(eid)
efiles = [ef for ef in spikeglx.glob_ephys_files(session_path, bin_exists=False)
if ef.get('lf', None)]
return [spikeglx.Reader(ef['lf']) for ef in efiles]
def _run(self, overwrite=False):
"""
Multiple steps. For each probe:
- Runs ks2 (skips if it already ran)
- synchronize the spike sorting
- output the probe description files
:param overwrite:
:return: list of files to be registered on database
"""
efiles = spikeglx.glob_ephys_files(self.session_path)
ap_files = [(ef.get("ap"), ef.get("label")) for ef in efiles
if "ap" in ef.keys()]
out_files = []
for ap_file, label in ap_files:
try:
ks2_dir = self._run_ks2(
ap_file) # runs ks2, skips if it already ran
probe_out_path = self.session_path.joinpath("alf", label)
probe_out_path.mkdir(parents=True, exist_ok=True)
spikes.ks2_to_alf(
ks2_dir,
bin_path=ap_file.parent,
out_path=probe_out_path,
bin_file=ap_file,
ampfactor=self._sample2v(ap_file),
)
out, _ = spikes.sync_spike_sorting(ap_file=ap_file,
out_path=probe_out_path)
out_files.extend(out)
# convert ks2_output into tar file and also register
# Make this in case spike sorting is in old raw_ephys_data folders, for new
# sessions it should already exist
tar_dir = self.session_path.joinpath('spike_sorters',
'ks2_matlab', label)
tar_dir.mkdir(parents=True, exist_ok=True)
out = spikes.ks2_to_tar(ks2_dir, tar_dir)
out_files.extend(out)
except BaseException:
_logger.error(traceback.format_exc())
self.status = -1
continue
probe_files = spikes.probes_description(self.session_path,
one=self.one)
return out_files + probe_files
def _run(self):
"""
Compress ephys files looking for `compress_ephys.flag` whithin the probes folder
Original bin file will be removed
The registration flag created contains targeted file names at the root of the session
"""
ephys_files = spikeglx.glob_ephys_files(self.session_path)
out_files = []
for ef in ephys_files:
for typ in ['ap', 'lf', 'nidq']:
bin_file = ef.get(typ)
if not bin_file:
continue
sr = spikeglx.Reader(bin_file)
if sr.is_mtscomp:
out_files.append(bin_file)
else:
_logger.info(f"Compressing binary file {bin_file}")
out_files.append(sr.compress_file(keep_original=False))
return out_files
def version3B(ses_path, display=True, linear=False, tol=2.5):
"""
From a session path with _spikeglx_sync arrays extraccted, locate ephys files for 3A and
outputs one sync.timestamps.probeN.npy file per acquired probe. By convention the reference
probe is the one with the most synchronisation pulses.
Assumes the _spikeglx_sync datasets are already extracted from binary data
:param ses_path:
:return: None
"""
ephys_files = spikeglx.glob_ephys_files(ses_path)
for ef in ephys_files:
ef['sync'] = alf.io.load_object(ef.path,
'_spikeglx_sync',
short_keys=True)
ef['sync_map'] = get_ibl_sync_map(ef, '3B')
nidq_file = [ef for ef in ephys_files if ef.get('nidq')]
ephys_files = [ef for ef in ephys_files if not ef.get('nidq')]
nprobes = len(ephys_files)
# should have at least 2 probes and only one nidq
if nprobes <= 1:
return True
assert (len(nidq_file) == 1)
nidq_file = nidq_file[0]
sync_nidq = _get_sync_fronts(nidq_file.sync,
nidq_file.sync_map['imec_sync'])
qc_all = True
for ef in ephys_files:
sync_probe = _get_sync_fronts(ef.sync, ef.sync_map['imec_sync'])
sr = _get_sr(ef)
assert (sync_nidq.times.size == sync_probe.times.size)
timestamps, qc = sync_probe_front_times(sync_probe.times,
sync_nidq.times,
sr,
display=display,
linear=linear,
tol=tol)
qc_all &= qc
_save_timestamps_npy(ef, timestamps)
return qc_all
def _get_task_sync(session_path):
"""
From 3A or 3B multiprobe session, returns the main probe (3A) or nidq sync pulses
with the attached channel map (default chmap if none)
:param session_path:
:return:
"""
def _get_probe_version_from_files(ephys_files):
if any([ef.get('nidq') for ef in ephys_files]):
return '3B'
else:
return '3A'
# round-up of all bin ephys files in the session, infer revision and get sync map
ephys_files = glob_ephys_files(session_path)
version = _get_probe_version_from_files(ephys_files)
sync_chmap = CHMAPS[version]
extract_sync(session_path, save=True)
# attach the sync information to each binary file found
for ef in ephys_files:
ef['sync'] = alf.io.load_object(ef.path,
'_spikeglx_sync',
short_keys=True)
ef['sync_map'] = ibllib.io.spikeglx.get_sync_map(ef['path'])
if version == '3A':
# the sync master is the probe with the most sync pulses
sync_box_ind = np.argmax([ef.sync.times.size for ef in ephys_files])
elif version == '3B':
# the sync master is the nidq breakout box
sync_box_ind = np.argmax(
[1 if ef.get('nidq') else 0 for ef in ephys_files])
sync = ephys_files[sync_box_ind].sync
return sync, sync_chmap
def version3A(ses_path, display=True, type='smooth', tol=2.1):
"""
From a session path with _spikeglx_sync arrays extracted, locate ephys files for 3A and
outputs one sync.timestamps.probeN.npy file per acquired probe. By convention the reference
probe is the one with the most synchronisation pulses.
Assumes the _spikeglx_sync datasets are already extracted from binary data
:param ses_path:
:param type: linear, exact or smooth
:return: bool True on a a successful sync
"""
ephys_files = spikeglx.glob_ephys_files(ses_path, bin_exists=False)
nprobes = len(ephys_files)
if nprobes == 1:
timestamps = np.array([[0., 0.], [1., 1.]])
sr = _get_sr(ephys_files[0])
out_files = _save_timestamps_npy(ephys_files[0], timestamps, sr)
return True, out_files
def get_sync_fronts(auxiliary_name):
d = Bunch({'times': [], 'nsync': np.zeros(nprobes, )})
# auxiliary_name: frame2ttl or right_camera
for ind, ephys_file in enumerate(ephys_files):
sync = alf.io.load_object(ephys_file.ap.parent,
'_spikeglx_sync',
short_keys=True)
sync_map = get_ibl_sync_map(ephys_file, '3A')
# exits if sync label not found for current probe
if auxiliary_name not in sync_map:
return
isync = np.in1d(sync['channels'],
np.array([sync_map[auxiliary_name]]))
# only returns syncs if we get fronts for all probes
if np.all(~isync):
return
d.nsync[ind] = len(sync.channels)
d['times'].append(sync['times'][isync])
return d
d = get_sync_fronts('frame2ttl')
if not d:
_logger.warning(
'Ephys sync: frame2ttl not detected on both probes, using camera sync'
)
d = get_sync_fronts('right_camera')
if not min([t[0] for t in d['times']]) > 0.2:
raise (
ValueError('Cameras started before ephys, no sync possible'))
# chop off to the lowest number of sync points
nsyncs = [t.size for t in d['times']]
if len(set(nsyncs)) > 1:
_logger.warning(
"Probes don't have the same number of synchronizations pulses")
d['times'] = np.r_[[t[:min(nsyncs)] for t in d['times']]].transpose()
# the reference probe is the one with the most sync pulses detected
iref = np.argmax(d.nsync)
# islave = np.setdiff1d(np.arange(nprobes), iref)
# get the sampling rate from the reference probe using metadata file
sr = _get_sr(ephys_files[iref])
qc_all = True
# output timestamps files as per ALF convention
for ind, ephys_file in enumerate(ephys_files):
if ind == iref:
timestamps = np.array([[0., 0.], [1., 1.]])
else:
timestamps, qc = sync_probe_front_times(d.times[:, ind],
d.times[:, iref],
sr,
display=display,
type=type,
tol=tol)
qc_all &= qc
out_files = _save_timestamps_npy(ephys_file, timestamps, sr)
return qc_all, out_files
def test_glob_ephys(self):
def dict_equals(d1, d2):
return all([l in d1 for l in d2]) and all([l in d2 for l in d1])
self.assertTrue(dict_equals(self.dict3a, spikeglx.glob_ephys_files(self.dir3a)))
self.assertTrue(dict_equals(self.dict3b, spikeglx.glob_ephys_files(self.dir3b)))
one = ONE()
# Get a specific session eID
eid = one.search(subject='ZM_2240', date_range='2020-01-22')[0]
# Define and load dataset types of interest
dtypes = [
'ephysData.raw.lf', 'ephysData.raw.meta', 'ephysData.raw.ch',
'ephysData.raw.sync'
]
one.load(eid, dataset_types=dtypes, download_only=True)
# Get the files information
session_path = one.path_from_eid(eid)
efiles = [
ef for ef in spikeglx.glob_ephys_files(session_path, bin_exists=False)
if ef.get('lf', None)
]
efile = efiles[0]['lf']
# === Option 2 === You can also input a file locally, e.g.
# efile = ('/datadisk/FlatIron/churchlandlab/Subjects/CSHL045/2020-02-26/001/'
# 'raw_ephys_data/probe01/_spikeglx_ephysData_g0_t0.imec.lf.cbin')
# === Read the files and get the data ===
sr = spikeglx.Reader(efile)
sync_file = sr.file_bin.parent.joinpath(
sr.file_bin.stem.replace('.lf', '.sync.npy'))
sync = np.load(sync_file)
sample2time = scipy.interpolate.interp1d(sync[:, 0] * sr.fs, sync[:, 1])