def get_stimOn_times(session_path, save=False, data=False, settings=False):
"""
Find the time of the statemachine command to turn on hte stim
(state stim_on start or rotary_encoder_event2)
Find the next frame change from the photodiodeafter that TS.
Screen is not displaying anything until then.
(Frame changes are in BNC1High and BNC1Low)
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
stimOn_times = get_stimOn_times_ge5(session_path, data=data)
else:
stimOn_times = get_stimOn_times_lt5(session_path, data=data)
if raw.save_bool(save, '_ibl_trials.stimOn_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.stimOn_times.npy')
np.save(fpath, np.array(stimOn_times))
return np.array(stimOn_times)
def extract_all(session_path, save=False, bpod_trials=None, settings=None):
if not bpod_trials:
bpod_trials = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
base = [
FeedbackType, ContrastLR, ProbabilityLeft, Choice, RepNum,
RewardVolume, FeedbackTimes, Intervals, ResponseTimes,
GoCueTriggerTimes, GoCueTimes
]
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
base.extend([
StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes,
StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes
])
else:
base.extend([IncludedTrials, ItiDuration, StimOnTimes])
out, fil = run_extractor_classes(base,
save=save,
session_path=session_path,
bpod_trials=bpod_trials,
settings=settings)
return out, fil
def get_feedback_times(session_path, save=False, data=False, settings=False):
"""
Get the times the water or error tone was delivered to the animal.
**Optional:** saves _ibl_trials.feedback_times.npy
Gets reward and error state init times vectors,
checks if theintersection of nans is empty, then
merges the 2 vectors.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
merge = get_feedback_times_ge5(session_path, data=data)
else:
merge = get_feedback_times_lt5(session_path, data=data)
if raw.save_bool(save, '_ibl_trials.feedback_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.feedback_times.npy')
np.save(fpath, merge)
return np.array(merge)
def _extract(self):
if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
trials_included = self.get_included_trials_ge5(
data=self.bpod_trials, settings=self.settings)
else:
trials_included = self.get_included_trials_lt5(data=self.bpod_trials)
return trials_included
def test_trials_extraction(self):
# extract all sessions
with tempfile.TemporaryDirectory() as tdir:
subjects_path = Path(tdir).joinpath('Subjects')
shutil.copytree(self.INIT_FOLDER, subjects_path)
for fil in subjects_path.rglob('_iblrig_taskData.raw*.jsonable'):
session_path = fil.parents[1]
# task running part
job = TrainingTrials(session_path, one=self.one)
job.run()
# check the trials objects
trials = alf.io.load_object(session_path / 'alf', 'trials')
self.assertTrue(alf.io.check_dimensions(trials) == 0)
settings = rawio.load_settings(session_path)
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
tkeys = TRIAL_KEYS_ge5
else:
tkeys = TRIAL_KEYS_lt5
self.assertTrue(set(trials.keys()) == set(tkeys))
# check the wheel object if the extraction didn't fail
if job.status != -1:
wheel = alf.io.load_object(session_path / 'alf', 'wheel')
self.assertTrue(alf.io.check_dimensions(wheel) == 0)
self.assertTrue(set(wheel.keys()) == set(WHEEL_KEYS))
"""
For this session only the downgoing front of a trial was detected, resulting in
an error for the gocuetime. The fix was to extract the downgoing front and
subtract 100ms.
"""
session_path = subjects_path / "CSHL_007/2019-07-31/001"
trials = alf.io.load_object(session_path / 'alf', 'trials')
self.assertTrue(np.all(np.logical_not(np.isnan(trials.goCue_times))))
def _extract(self):
# Version check
if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
merge = self.get_feedback_times_ge5(self.session_path, data=self.bpod_trials)
else:
merge = self.get_feedback_times_lt5(self.session_path, data=self.bpod_trials)
return np.array(merge)
def _extract(self):
if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
goCue = np.array([tr['behavior_data']['States timestamps']
['play_tone'][0][0] for tr in self.bpod_trials])
else:
goCue = np.array([tr['behavior_data']['States timestamps']
['closed_loop'][0][0] for tr in self.bpod_trials])
return goCue
def _extract(self):
"""
Find the time of the state machine command to turn on the stim
(state stim_on start or rotary_encoder_event2)
Find the next frame change from the photodiode after that TS.
Screen is not displaying anything until then.
(Frame changes are in BNC1 High and BNC1 Low)
"""
# Version check
if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
stimOn_times = self.get_stimOn_times_ge5(self.session_path, data=self.bpod_trials)
else:
stimOn_times = self.get_stimOn_times_lt5(self.session_path, data=self.bpod_trials)
return np.array(stimOn_times)
def extract_all(session_path, save=False, data=False, settings=False):
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Common to all versions
feedbackType = get_feedbackType(session_path, save=save, data=data, settings=settings)
contrastLeft, contrastRight = get_contrastLR(
session_path, save=save, data=data, settings=settings)
probabilityLeft = get_probabilityLeft(
session_path, save=save, data=data, settings=settings)
choice = get_choice(session_path, save=save, data=data, settings=settings)
rewardVolume = get_rewardVolume(session_path, save=save, data=data, settings=settings)
feedback_times = get_feedback_times(session_path, save=save, data=data, settings=settings)
stimOn_times = get_stimOn_times(session_path, save=save, data=data, settings=settings)
intervals = get_intervals(session_path, save=save, data=data, settings=settings)
response_times = get_response_times(session_path, save=save, data=data, settings=settings)
go_cue_trig_times = get_goCueTrigger_times(
session_path, save=save, data=data, settings=settings)
go_cue_times = get_goCueOnset_times(session_path, save=save, data=data, settings=settings)
camera_timestamps = get_camera_timestamps(
session_path, save=save, data=data, settings=settings)
out = {'feedbackType': feedbackType,
'contrastLeft': contrastLeft,
'contrastRight': contrastRight,
'probabilityLeft': probabilityLeft,
'session_path': session_path,
'choice': choice,
'rewardVolume': rewardVolume,
'feedback_times': feedback_times,
'stimOn_times': stimOn_times,
'intervals': intervals,
'response_times': response_times,
'camera_timestamps': camera_timestamps,
'goCue_times': go_cue_times,
'goCueTrigger_times': go_cue_trig_times}
# Version specific extractions
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
out['stimOnTrigger_times'] = get_stimOnTrigger_times(
session_path, save=save, data=data, settings=settings)
out['included'] = get_included_trials(
session_path, save=save, data=data, settings=settings)
else:
out['iti_dur'] = get_iti_duration(session_path, save=save, data=data, settings=settings)
return out
def load_encoder_positions(session_path, settings=False):
"""
Load Rotary Encoder (RE) positions from raw data file within a session path.
Assumes that a folder called "raw_behavior_data" exists in folder.
Positions are RE ticks [-512, 512] == [-180º, 180º]
0 == trial stim init position
Positive nums are rightwards movements (mouse) or RE CW (mouse)
Variable line number, depends on movements.
Raw datafile Columns:
Position, RE timestamp, RE Position, Bonsai Timestamp
Position is always equal to 'Position' so this column was dropped.
>>> data.columns
>>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'
're_pos', # Rotary Encoder position (ticks) 'numpy.int64'
'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp'
# pd.to_datetime(data.bns_ts) to work in datetimes
:param session_path: Absolute path of session folder
:type session_path: str
:return: dataframe w/ 3 cols and N positions
:rtype: Pandas.DataFrame
"""
if session_path is None:
return
path = Path(session_path).joinpath("raw_behavior_data")
path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None)
if not settings:
settings = load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# auto-detect old files when version is not labeled
with open(path) as fid:
line = fid.readline()
if line.startswith('Position'):
settings = {'IBLRIG_VERSION_TAG': '0.0.0'}
if not path:
_logger.warning("No data loaded: could not find raw encoderPositions file")
return None
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
return _load_encoder_positions_file_ge5(path)
else:
return _load_encoder_positions_file_lt5(path)
def get_included_trials(session_path, save=False, data=False, settings=False):
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
trials_included = get_included_trials_ge5(session_path, data=data, settings=settings)
else:
trials_included = get_included_trials_lt5(session_path, data=data)
if raw.save_bool(save, '_ibl_trials.included'):
fpath = Path(session_path).joinpath('alf', '_ibl_trials.included.npy')
np.save(fpath, trials_included)
return trials_included
def get_goCueTrigger_times(session_path,
save=False,
data=False,
settings=False):
"""
Get trigger times of goCue from state machine.
Current software solution for triggering sounds uses PyBpod soft codes.
Delays can be in the order of 10's of ms. This is the time when the command
to play the sound was executed. To measure accurate time, either getting the
sound onset from xonar soundcard sync pulse (latencies may vary).
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
goCue = np.array([
tr['behavior_data']['States timestamps']['play_tone'][0][0]
for tr in data
])
else:
goCue = np.array([
tr['behavior_data']['States timestamps']['closed_loop'][0][0]
for tr in data
])
if raw.save_bool(save, '_ibl_trials.goCue_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.goCueTrigger_times.npy')
np.save(fpath, goCue)
return goCue
def extract_all(session_path, save=False, bpod_trials=None, settings=None):
"""Extract trials and wheel data.
For task versions >= 5.0.0, outputs wheel data and trials.table dataset (+ some extra datasets)
Parameters
----------
session_path : str, pathlib.Path
The path to the session
save : bool
If true save the data files to ALF
bpod_trials : list of dicts
The Bpod trial dicts loaded from the _iblrig_taskData.raw dataset
settings : dict
The Bpod settings loaded from the _iblrig_taskSettings.raw dataset
Returns
-------
A list of extracted data and a list of file paths if save is True (otherwise None)
"""
if not bpod_trials:
bpod_trials = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
base = [RepNum, GoCueTriggerTimes]
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
# We now extract a single trials table
base.extend([
StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes,
ErrorCueTriggerTimes, TrialsTable, PhasePosQuiescence
])
else:
base.extend([
Intervals, Wheel, FeedbackType, ContrastLR, ProbabilityLeft, Choice, IncludedTrials, ItiDuration,
StimOnTimes_deprecated, RewardVolume, FeedbackTimes, ResponseTimes, GoCueTimes, PhasePosQuiescence
])
out, fil = run_extractor_classes(
base, save=save, session_path=session_path, bpod_trials=bpod_trials, settings=settings)
return out, fil
def load_encoder_events(session_path, settings=False):
"""
Load Rotary Encoder (RE) events raw data file.
Assumes that a folder called "raw_behavior_data" exists in folder.
On each trial the RE sends 3 events to Bonsai 1 - meaning trial start/turn
off the stim; 2 - meaning show the current trial stimulus; and 3 - meaning
begin the closed loop making the stim move with the RE. These events are
triggered by the state machine in the corrensponding states: trial_start,
stim_on, closed_loop
Raw datafile Columns:
Event, RE timestamp, Source, data, Bonsai Timestamp
Event is always equal 'Event' Source is always equal 'StateMachine'. For
this reason these columns are dropped.
>>> data.columns
>>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'
'sm_ev', # State Machine Event 'numpy.int64'
'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp'
# pd.to_datetime(data.bns_ts) to work in datetimes
:param session_path: [description]
:type session_path: [type]
:return: dataframe w/ 3 cols and (ntrials * 3) lines
:rtype: Pandas.DataFrame
"""
if session_path is None:
return
path = Path(session_path).joinpath("raw_behavior_data")
path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None)
if not settings:
settings = load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
if not path:
return None
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
return _load_encoder_events_file_ge5(path)
else:
return _load_encoder_events_file_lt5(path)
def test_partial_extraction(self):
ex = TaskQCExtractor(self.session_path,
lazy=True,
one=self.one,
bpod_only=True)
ex.extract_data()
expected = [
'contrastLeft', 'contrastRight', 'phase', 'position',
'probabilityLeft', 'quiescence', 'stimOn_times'
]
expected_5up = [
'contrast', 'errorCueTrigger_times', 'itiIn_times',
'stimFreezeTrigger_times', 'stimFreeze_times',
'stimOffTrigger_times', 'stimOff_times', 'stimOnTrigger_times'
]
if version.ge(ex.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
expected += expected_5up
self.assertTrue(set(expected).issubset(set(ex.data.keys())))
def load_encoder_events(session_path, settings=False):
"""
Load Rotary Encoder (RE) events raw data file.
Assumes that a folder called "raw_behavior_data" exists in folder.
Events number correspond to following bpod states:
1: correct / hide_stim
2: stim_on
3: closed_loop
4: freeze_error / freeze_correct
>>> data.columns
>>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'
'sm_ev', # State Machine Event 'numpy.int64'
'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp'
# pd.to_datetime(data.bns_ts) to work in datetimes
:param session_path: [description]
:type session_path: [type]
:return: dataframe w/ 3 cols and (ntrials * 3) lines
:rtype: Pandas.DataFrame
"""
if session_path is None:
return
path = Path(session_path).joinpath("raw_behavior_data")
path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None)
if not settings:
settings = load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# auto-detect old files when version is not labeled
with open(path) as fid:
line = fid.readline()
if line.startswith('Event') and 'StateMachine' in line:
settings = {'IBLRIG_VERSION_TAG': '0.0.0'}
if not path:
return None
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
return _load_encoder_events_file_ge5(path)
else:
return _load_encoder_events_file_lt5(path)
def extract_all(session_path, save=False, bpod_trials=False, settings=False, extra_classes=None):
"""
Same as training_trials.extract_all except...
- there is no RepNum
- ContrastLR is extracted differently
- IncludedTrials is only extracted for 5.0.0 or greater
:param session_path:
:param save:
:param bpod_trials:
:param settings:
:param extra_classes: additional BaseBpodTrialsExtractor subclasses for custom extractions
:return:
"""
if not bpod_trials:
bpod_trials = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
base = [GoCueTriggerTimes]
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
# We now extract a single trials table
base.extend([
StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes,
TrialsTableBiased, IncludedTrials, PhasePosQuiescence
])
else:
base.extend([
Intervals, Wheel, FeedbackType, ContrastLR, ProbabilityLeft, Choice, ItiDuration,
StimOnTimes_deprecated, RewardVolume, FeedbackTimes, ResponseTimes, GoCueTimes, PhasePosQuiescence
])
if extra_classes:
base.extend(extra_classes)
out, fil = run_extractor_classes(
base, save=save, session_path=session_path, bpod_trials=bpod_trials, settings=settings)
return out, fil
def assert_g(self, v0, v_):
self.assertFalse(version.eq(v0, v_))
self.assertTrue(version.ge(v0, v_))
self.assertFalse(version.le(v0, v_))
self.assertTrue(version.gt(v0, v_))
self.assertFalse(version.lt(v0, v_))
