def from_path(session_path, force=False, save=True):
"""
Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001')
:param force: (False) overwrite existing files
:param save: (True) boolean or list of ALF file names to extract
:return: None
"""
logger_.info('Extracting ' + str(session_path))
extractor_type = extractors_exist(session_path)
if is_extracted(session_path) and not force:
logger_.info(f"Session {session_path} already extracted.")
return
if extractor_type == 'training':
settings, data = raw.load_bpod(session_path)
logger_.info('training session on ' + settings['PYBPOD_BOARD'])
training_trials.extract_all(session_path, data=data, save=save)
training_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'biased':
settings, data = raw.load_bpod(session_path)
logger_.info('biased session on ' + settings['PYBPOD_BOARD'])
biased_trials.extract_all(session_path, data=data, save=save)
biased_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'ephys':
data = raw.load_data(session_path)
ephys_trials.extract_all(session_path, data=data, save=save)
ephys_fpga.extract_all(session_path, save=save)
def from_path(session_path, force=False, save=True):
"""
Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001')
:param force: (False) overwrite existing files
:param save: (True) boolean or list of ALF file names to extract
:return: None
"""
extractor_type = get_session_extractor_type(session_path)
logger_.info(f"Extracting {session_path} as {extractor_type}")
if is_extracted(session_path) and not force:
logger_.info(f"Session {session_path} already extracted.")
return
if extractor_type == 'training':
settings, data = rawio.load_bpod(session_path)
logger_.info('training session on ' + settings['PYBPOD_BOARD'])
training_trials.extract_all(session_path, data=data, save=save)
training_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'biased':
settings, data = rawio.load_bpod(session_path)
logger_.info('biased session on ' + settings['PYBPOD_BOARD'])
biased_trials.extract_all(session_path, data=data, save=save)
biased_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'ephys':
data = rawio.load_data(session_path)
logger_.info('extract BPOD for ephys session')
ephys_trials.extract_all(session_path, data=data, save=save)
logger_.info('extract FPGA information for ephys session')
tmax = data[-1]['behavior_data']['States timestamps']['exit_state'][0][-1] + 60
ephys_fpga.extract_all(session_path, save=save, tmax=tmax)
if extractor_type == 'sync_ephys':
ephys_fpga.extract_sync(session_path, save=save)
def compare_wheel_fpga_behaviour(session_path, display=DISPLAY):
alf_path = session_path.joinpath('alf')
shutil.rmtree(alf_path, ignore_errors=True)
sync, chmap = ephys_fpga.get_main_probe_sync(session_path,
bin_exists=False)
fpga_t, fpga_pos = ephys_fpga.extract_wheel_sync(sync, chmap=chmap)
bpod_t, bpod_pos = training_wheel.get_wheel_position(session_path,
display=display)
data, _ = ephys_fpga.extract_all(session_path)
bpod2fpga = scipy.interpolate.interp1d(data['intervals_bpod'][:, 0],
data['intervals'][:, 0],
fill_value="extrapolate")
# resample both traces to the same rate and compute correlation coeff
bpod_t = bpod2fpga(bpod_t)
tmin = max([np.min(fpga_t), np.min(bpod_t)])
tmax = min([np.max(fpga_t), np.max(bpod_t)])
wheel = {'tscale': np.arange(tmin, tmax, 0.01)}
wheel['fpga'] = scipy.interpolate.interp1d(fpga_t,
fpga_pos)(wheel['tscale'])
wheel['bpod'] = scipy.interpolate.interp1d(bpod_t,
bpod_pos)(wheel['tscale'])
if display:
plt.figure()
plt.plot(fpga_t - bpod2fpga(0), fpga_pos, '*')
plt.plot(bpod_t - bpod2fpga(0), bpod_pos, '.')
raw_wheel = {
'fpga_t': fpga_t,
'fpga_pos': fpga_pos,
'bpod_t': bpod_t,
'bpod_pos': bpod_pos
}
return raw_wheel, wheel
def extract_data(self):
"""Extracts and loads behaviour data for QC
NB: partial extraction when bpod_only attribute is False requires intervals and
intervals_bpod to be assigned to the data attribute before calling this function.
:return:
"""
self.log.info(f"Extracting session: {self.session_path}")
self.type = self.type or get_session_extractor_type(self.session_path)
self.wheel_encoding = 'X4' if (self.type == 'ephys' and not self.bpod_only) else 'X1'
if not self.raw_data:
self.load_raw_data()
# Run extractors
if self.type == 'ephys' and not self.bpod_only:
data, _ = ephys_fpga.extract_all(self.session_path)
bpod2fpga = interp1d(data['intervals_bpod'][:, 0], data['table']['intervals_0'],
fill_value='extrapolate')
# Add Bpod wheel data
re_ts, pos = get_wheel_position(self.session_path, self.raw_data)
data['wheel_timestamps_bpod'] = bpod2fpga(re_ts)
data['wheel_position_bpod'] = pos
else:
kwargs = dict(save=False, bpod_trials=self.raw_data, settings=self.settings)
trials, wheel, _ = bpod_trials.extract_all(self.session_path, **kwargs)
n_trials = np.unique(list(map(lambda k: trials[k].shape[0], trials)))[0]
if self.type == 'habituation':
data = trials
data['position'] = np.array([t['position'] for t in self.raw_data])
data['phase'] = np.array([t['stim_phase'] for t in self.raw_data])
# Nasty hack to trim last trial due to stim off events happening at trial num + 1
data = {k: v[:n_trials] for k, v in data.items()}
else:
data = {**trials, **wheel}
# Update the data attribute with extracted data
self.data = self.rename_data(data)
def _qc_from_path(sess_path, display=True):
WHEEL = False
sess_path = Path(sess_path)
temp_alf_folder = sess_path.joinpath('fpga_test', 'alf')
temp_alf_folder.mkdir(parents=True, exist_ok=True)
raw_trials = raw_data_loaders.load_data(sess_path)
tmax = raw_trials[-1]['behavior_data']['States timestamps']['exit_state'][
0][-1] + 60
sync, chmap = ephys_fpga._get_main_probe_sync(sess_path, bin_exists=False)
_ = ephys_fpga.extract_all(sess_path,
output_path=temp_alf_folder,
save=True)
# check that the output is complete
fpga_trials = ephys_fpga.extract_behaviour_sync(
sync,
output_path=temp_alf_folder,
tmax=tmax,
chmap=chmap,
save=True,
display=display)
# align with the bpod
bpod2fpga = ephys_fpga.align_with_bpod(temp_alf_folder.parent)
alf_trials = alf.io.load_object(temp_alf_folder, 'trials')
shutil.rmtree(temp_alf_folder)
# do the QC
qcs, qct = qc_fpga_task(fpga_trials, alf_trials)
# do the wheel part
if WHEEL:
bpod_wheel = training_wheel.get_wheel_data(sess_path, save=False)
fpga_wheel = ephys_fpga.extract_wheel_sync(sync,
chmap=chmap,
save=False)
if display:
import matplotlib.pyplot as plt
t0 = max(np.min(bpod2fpga(bpod_wheel['re_ts'])),
np.min(fpga_wheel['re_ts']))
dy = np.interp(
t0, fpga_wheel['re_ts'], fpga_wheel['re_pos']) - np.interp(
t0, bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'])
fix, axes = plt.subplots(nrows=2, sharex='all', sharey='all')
# axes[0].plot(t, pos), axes[0].title.set_text('Extracted')
axes[0].plot(bpod2fpga(bpod_wheel['re_ts']),
bpod_wheel['re_pos'] + dy)
axes[0].plot(fpga_wheel['re_ts'], fpga_wheel['re_pos'])
axes[0].title.set_text('FPGA')
axes[1].plot(bpod2fpga(bpod_wheel['re_ts']),
bpod_wheel['re_pos'] + dy)
axes[1].title.set_text('Bpod')
return alf.io.dataframe({**fpga_trials, **alf_trials, **qct})
def __init__(self, session_path, bpod_only=False, local=False):
"""
Loads and extracts the QC data for a given session path
:param session_path: A str or Path to a Bpod session
:param bpod_only: When True all data is extracted from Bpod instead of FPGA for ephys
"""
super().__init__(session_path, one=one, log=_logger)
if local:
dsets, out_files = ephys_fpga.extract_all(session_path, save=True)
self.extractor = TaskQCExtractor(session_path, lazy=True, one=one)
# Extract extra datasets required for QC
self.extractor.data = dsets
self.extractor.extract_data()
# Aggregate and update Alyx QC fields
self.run(update=False)
else:
self.load_data(bpod_only=bpod_only)
self.compute()
self.n_trials = self.extractor.data['intervals'].shape[0]
self.wheel_data = {
're_pos': self.extractor.data.pop('wheel_position'),
're_ts': self.extractor.data.pop('wheel_timestamps')
}
# Print failed
outcome, results, outcomes = self.compute_session_status()
map = {k: [] for k in set(outcomes.values())}
for k, v in outcomes.items():
map[v].append(k[6:])
for k, v in map.items():
if k == 'PASS':
continue
print(f'The following checks were labelled {k}:')
print('\n'.join(v), '\n')
# Make DataFrame from the trail level metrics
def get_trial_level_failed(d):
new_dict = {
k[6:]: v
for k, v in d.items()
if isinstance(v, Sized) and len(v) == self.n_trials
}
return pd.DataFrame.from_dict(new_dict)
self.frame = get_trial_level_failed(self.metrics)
self.frame['intervals_0'] = self.extractor.data['intervals'][:, 0]
self.frame['intervals_1'] = self.extractor.data['intervals'][:, 1]
self.frame.insert(loc=0, column='trial_no', value=self.frame.index)
def _run(self):
dsets, out_files = ephys_fpga.extract_all(self.session_path, save=True)
self._behaviour_criterion()
# Run the task QC
qc = TaskQC(self.session_path, one=self.one, log=_logger)
qc.extractor = TaskQCExtractor(self.session_path,
lazy=True,
one=qc.one)
# Extract extra datasets required for QC
qc.extractor.data = dsets
qc.extractor.extract_data()
# Aggregate and update Alyx QC fields
qc.run(update=True)
return out_files
def test_frame2ttl_flicker(self):
from ibllib.io.extractors import ephys_fpga
from ibllib.qc.task_metrics import TaskQC
from ibllib.qc.task_extractors import TaskQCExtractor
init_path = self.data_path.joinpath("ephys", "ephys_choice_world_task")
session_path = init_path.joinpath("ibl_witten_27/2021-01-21/001")
dsets, out_files = ephys_fpga.extract_all(session_path, save=True)
# Run the task QC
qc = TaskQC(session_path, one=None)
qc.extractor = TaskQCExtractor(session_path, lazy=True, one=None)
# Extr+act extra datasets required for QC
qc.extractor.data = dsets
qc.extractor.extract_data()
# Aggregate and update Alyx QC fields
_, myqc = qc.run(update=False)
# from ibllib.misc import pprint
# pprint(myqc)
assert myqc['_task_stimOn_delays'] > 0.9 # 0.6176
assert myqc["_task_stimFreeze_delays"] > 0.9
assert myqc["_task_stimOff_delays"] > 0.9
assert myqc["_task_stimOff_itiIn_delays"] > 0.9
assert myqc["_task_stimOn_delays"] > 0.9
assert myqc["_task_stimOn_goCue_delays"] > 0.9
def _task_extraction_assertions(self, session_path):
alf_path = session_path.joinpath('alf')
shutil.rmtree(alf_path, ignore_errors=True)
# this gets the full output
ephys_fpga.extract_all(session_path, save=True, bin_exists=False)
# check that the output is complete
for f in BPOD_FILES:
self.assertTrue(alf_path.joinpath(f).exists())
# check that the output is complete
for f in FPGA_FILES:
self.assertTrue(alf_path.joinpath(f).exists())
# check dimensions after alf load
alf_trials = alf.io.load_object(alf_path, 'trials')
self.assertTrue(alf.io.check_dimensions(alf_trials) == 0)
# go deeper and check the internal fpga trials structure consistency
sync, chmap = ephys_fpga.get_main_probe_sync(session_path,
bin_exists=False)
fpga_trials = ephys_fpga.extract_behaviour_sync(sync, chmap)
# check dimensions
self.assertEqual(alf.io.check_dimensions(fpga_trials), 0)
# check that the stimOn < stimFreeze < stimOff
self.assertTrue(
np.all(fpga_trials['stimOn_times'][:-1] <
fpga_trials['stimOff_times'][:-1]))
self.assertTrue(
np.all(fpga_trials['stimFreeze_times'][:-1] <
fpga_trials['stimOff_times'][:-1]))
# a trial is either an error-nogo or a reward
self.assertTrue(
np.all(
np.isnan(fpga_trials['valveOpen_times'][:-1] *
fpga_trials['errorCue_times'][:-1])))
self.assertTrue(
np.all(
np.logical_xor(np.isnan(fpga_trials['valveOpen_times'][:-1]),
np.isnan(fpga_trials['errorCue_times'][:-1]))))
# do the task qc
# tqc_ephys.extractor.settings['PYBPOD_PROTOCOL']
from ibllib.qc.task_extractors import TaskQCExtractor
ex = TaskQCExtractor(session_path,
lazy=True,
one=None,
bpod_only=False)
ex.data = fpga_trials
ex.extract_data()
from ibllib.qc.task_metrics import TaskQC
# '/mnt/s0/Data/IntegrationTests/ephys/ephys_choice_world_task/CSP004/2019-11-27/001'
tqc_ephys = TaskQC(session_path)
tqc_ephys.extractor = ex
_, res_ephys = tqc_ephys.run(bpod_only=False, download_data=False)
tqc_bpod = TaskQC(session_path)
_, res_bpod = tqc_bpod.run(bpod_only=True, download_data=False)
# for a swift comparison using variable explorer
# import pandas as pd
# df = pd.DataFrame([[res_bpod[k], res_ephys[k]] for k in res_ephys], index=res_ephys.keys())
ok = True
for k in res_ephys:
if k == "_task_response_feedback_delays":
continue
if (np.abs(res_bpod[k] - res_ephys[k]) > .2):
ok = False
print(f"{k} bpod: {res_bpod[k]}, ephys: {res_ephys[k]}")
assert ok
shutil.rmtree(alf_path, ignore_errors=True)
def _run(self):
dsets, out_files = ephys_fpga.extract_all(self.session_path, save=True)
return out_files