def setUp(self):
self.main_path = Path(__file__).parent
self.training_lt5 = {
'path': self.main_path / 'data' / 'session_training_lt5'
}
self.biased_lt5 = {
'path': self.main_path / 'data' / 'session_biased_lt5'
}
self.training_ge5 = {
'path': self.main_path / 'data' / 'session_training_ge5'
}
self.biased_ge5 = {
'path': self.main_path / 'data' / 'session_biased_ge5'
}
self.training_lt5['ntrials'] = len(
raw.load_data(self.training_lt5['path']))
self.biased_lt5['ntrials'] = len(raw.load_data(
self.biased_lt5['path']))
self.training_ge5['ntrials'] = len(
raw.load_data(self.training_ge5['path']))
self.biased_ge5['ntrials'] = len(raw.load_data(
self.biased_ge5['path']))
# turn off logging for unit testing as we will purposedly go into warning/error cases
self.wheel_ge5_path = self.main_path / 'data' / 'wheel_ge5'
self.wheel_lt5_path = self.main_path / 'data' / 'wheel_lt5'
self.logger = logging.getLogger('ibllib')
def from_path(session_path, force=False, save=True):
"""
Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001')
force: (False) overwrite existing files
save: (True) boolean or list of ALF file names to extract
"""
logger_.info('Extracting ' + str(session_path))
extractor_type = extractors_exist(session_path)
if is_extracted(session_path) and not force:
logger_.info(f"Session {session_path} already extracted.")
return
if extractor_type == 'training':
data = raw.load_data(session_path)
training_trials.extract_all(session_path, data=data, save=save)
training_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'biased':
data = raw.load_data(session_path)
biased_trials.extract_all(session_path, data=data, save=save)
biased_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'Reward':
data = raw.load_data(session_path)
biased_Reward_trials.extract_all(session_path, data=data, save=save)
biased_Reward_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
def setUp(self):
self.session_path = Path(__file__).parent / 'data' / 'session'
self.data = loaders.load_data(self.session_path)
self.session_path_biased = Path(
__file__).parent / 'data' / 'session_biased'
self.data_biased = loaders.load_data(self.session_path_biased)
self.wheel_path = Path(__file__).parent / 'data' / 'wheel'
# turn off logging for unit testing as we will purposely go into warning/error cases
self.logger = logging.getLogger('ibllib').setLevel(50)
def get_response_times(session_path, save=False, data=False):
"""
Time (in absolute seconds from session start) when a response was recorded.
**Optional:** saves _ibl_trials.response_times.npy
Uses the timestamp of the end of the closed_loop state.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
rt = np.array([
tr['behavior_data']['States timestamps']['closed_loop'][0][1]
for tr in data
])
if raw.save_bool(save, '_ibl_trials.response_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.response_times.npy')
np.save(fpath, rt)
return rt
def get_deadTime(session_path, save=False, data=False):
"""
Get the time between state machine exit and restart of next trial.
Uses the corrected Trial start and Trial end timpestamp values form PyBpod.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
starts = [t['behavior_data']['Trial start timestamp'] for t in data]
ends = [t['behavior_data']['Trial end timestamp'] for t in data]
# trial_len = np.array(ends) - np.array(starts)
deadTime = np.array(starts)[1:] - np.array(ends)[:-1]
deadTime = np.append(np.array([0]), deadTime)
if raw.save_bool(save, '_ibl_trials.deadTime.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.deadTime.npy')
np.save(fpath, deadTime)
return deadTime
def get_iti_duration(session_path, save=False, data=False):
"""
Calculate duration of iti from state timestamps.
**Optional:** saves _ibl_trials.iti_duration.npy
Uses Trial end timestamp and get_response_times to calculate iti.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
rt = get_response_times(session_path, save=False, data=False)
ends = np.array([t['behavior_data']['Trial end timestamp'] for t in data])
iti_dur = ends - rt
if raw.save_bool(save, '_ibl_trials.itiDuration.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.itiDuration.npy')
np.save(fpath, iti_dur)
return iti_dur
def get_intervals(session_path, save=False, data=False):
"""
Trial start to trial end. Trial end includes 1 or 2 seconds of iti depending
on if the trial was correct or not.
TODO: Nick suggested the that the iti be removed from this. In this case the
end of a trial would be the same as the response time.
Also consider adding _ibl_trials.iti and _ibl_trials.deadTime
**Optional:** saves _ibl_trials.intervals.npy
Uses the corrected Trial start and Trial end timpestamp values form PyBpod.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: 2D numpy.ndarray (col0 = start, col1 = end)
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
starts = [t['behavior_data']['Trial start timestamp'] for t in data]
ends = [t['behavior_data']['Trial end timestamp'] for t in data]
intervals = np.array([starts, ends]).T
if raw.save_bool(save, '_ibl_trials.intervals.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.intervals.npy')
np.save(fpath, intervals)
return intervals
def get_rewardVolume(session_path, save=False, data=False):
"""
Load reward volume delivered for each trial.
**Optional:** saves _ibl_trials.rewardVolume.npy
Uses reward_current to accumulate the amount of
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('int64')
"""
if not data:
data = raw.load_data(session_path)
trial_volume = [
x['reward_amount'] if x['trial_correct'] else 0 for x in data
]
rewardVolume = np.array(trial_volume).astype(np.float64)
assert len(rewardVolume) == len(data)
if raw.save_bool(save, '_ibl_trials.rewardVolume.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.rewardVolume.npy')
np.save(fpath, rewardVolume)
return rewardVolume
def get_included_trials_lt5(session_path, data=False):
if not data:
data = raw.load_data(session_path)
trials_included = np.array([True for t in data])
return trials_included
def get_intervals(session_path, save=False, data=False, settings=False):
"""
Trial start to trial end. Trial end includes 1 or 2 seconds after feedback,
(depending on the feedback) and 0.5 seconds of iti.
**Optional:** saves _ibl_trials.intervals.npy
Uses the corrected Trial start and Trial end timpestamp values form PyBpod.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: 2D numpy.ndarray (col0 = start, col1 = end)
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
starts = [t['behavior_data']['Trial start timestamp'] for t in data]
ends = [t['behavior_data']['Trial end timestamp'] for t in data]
intervals = np.array([starts, ends]).T
if raw.save_bool(save, '_ibl_trials.intervals.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.intervals.npy')
np.save(fpath, intervals)
return intervals
def get_stimOn_times(session_path, save=False, data=False, settings=False):
"""
Find the time of the statemachine command to turn on hte stim
(state stim_on start or rotary_encoder_event2)
Find the next frame change from the photodiodeafter that TS.
Screen is not displaying anything until then.
(Frame changes are in BNC1High and BNC1Low)
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
stimOn_times = get_stimOn_times_ge5(session_path, data=data)
else:
stimOn_times = get_stimOn_times_lt5(session_path, data=data)
if raw.save_bool(save, '_ibl_trials.stimOn_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.stimOn_times.npy')
np.save(fpath, np.array(stimOn_times))
return np.array(stimOn_times)
def full_bandit_fix(ses):
data = raw.load_data(ses)
choices = np.zeros(len(data))
choices[:] = np.nan
opto = np.zeros(len(data))
opto[:] = np.nan
left_reward = np.zeros(len(data))
left_reward[:] = np.nan
right_reward = np.zeros(len(data))
right_reward[:] = np.nan
for i in np.arange(len(data)):
if 'RotaryEncoder1_1' in data[i]['behavior_data']['Events timestamps']:
choices[i] = -1
if 'RotaryEncoder1_2' in data[i]['behavior_data']['Events timestamps']:
choices[i] = 1
if ('RotaryEncoder1_2' in data[i]['behavior_data']['Events timestamps']) \
& ('RotaryEncoder1_1' in data[i]['behavior_data']['Events timestamps']):
# Choose the earlier event, since that one caused reward
left_time = data[i]['behavior_data']['Events timestamps'][
'RotaryEncoder1_2'][0]
right_time = data[i]['behavior_data']['Events timestamps'][
'RotaryEncoder1_1'][0]
if left_time > right_time:
choices[i] = -1
if right_time > left_time:
choices[i] = 1
np.save(ses + '/alf/_ibl_trials.choice.npy', choices)
def test_groom_pin_state(self):
# ibl_witten_27\2021-01-14\001 # Can't assign a pin state
# CSK-im-002\2021-01-16\001 # Another example
root = self.data_path
session_path = root.joinpath('ephys', 'ephys_choice_world_task',
'ibl_witten_27', '2021-01-21', '001')
_, ts = raw.load_camera_ssv_times(session_path, 'left')
_, (*_, gpio) = raw.load_embedded_frame_data(session_path, 'left')
bpod_trials = raw.load_data(session_path)
_, audio = raw.load_bpod_fronts(session_path, bpod_trials)
# NB: syncing the timestamps to the audio doesn't work very well but we don't need it to
# for the extraction, so long as the audio and GPIO fronts match.
gpio, audio, _ = camio.groom_pin_state(gpio, audio, ts)
# Do some checks
self.assertEqual(gpio['indices'].size, audio['times'].size)
expected = np.array([164179, 164391, 164397, 164900, 164906], dtype=int)
np.testing.assert_array_equal(gpio['indices'][-5:], expected)
expected = np.array([2734.4496, 2737.9659, 2738.0659, 2746.4488, 2746.5488])
np.testing.assert_array_almost_equal(audio['times'][-5:], expected)
# Verify behaviour when audio and GPIO match in size
_, audio_, _ = camio.groom_pin_state(gpio, audio, ts, take='nearest', tolerance=.5)
self.assertEqual(audio, audio_)
# Verify behaviour when there are GPIO fronts beyond number of video frames
ts_short = ts[:gpio['indices'].max() - 10]
gpio_, *_ = camio.groom_pin_state(gpio, audio, ts_short)
self.assertFalse(np.any(gpio_['indices'] >= ts.size))
def get_probabilityLeft(session_path, save=False, data=False, settings=False):
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None:
settings = {"IBLRIG_VERSION_TAG": "100.0.0"}
elif settings["IBLRIG_VERSION_TAG"] == "":
settings.update({"IBLRIG_VERSION_TAG": "100.0.0"})
num = settings.get("PRELOADED_SESSION_NUM", None)
if num is None:
num = settings.get("PREGENERATED_SESSION_NUM", None)
if num is None:
fn = settings.get('SESSION_LOADED_FILE_PATH', None)
fn = PureWindowsPath(fn).name
num = ''.join([d for d in fn if d.isdigit()])
if num == '':
raise ValueError("Can't extract left probability behaviour.")
# Load the pregenerated file
sessions_folder = Path(raw.__file__).parent.joinpath(
'extractors', 'ephys_sessions')
fname = f"session_{num}_ephys_pcqs.npy"
pcqsp = np.load(sessions_folder.joinpath(fname))
pLeft = pcqsp[:, 4]
pLeft = pLeft[:len(data)]
if raw.save_bool(save, "_ibl_trials.probabilityLeft.npy"):
lpath = Path(session_path).joinpath("alf",
"_ibl_trials.probabilityLeft.npy")
np.save(lpath, pLeft)
return pLeft
def get_contrastLR(session_path, save=False, data=False):
"""
Get left and right contrasts from raw datafile
**Optional:** save _ibl_trials.contrastLeft.npy and
_ibl_trials.contrastRight.npy to alf folder.
Uses signed_contrast to create left and right contrast vectors.
:param session_path: absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:type save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
contrastLeft = np.array([t['signed_contrast'] for t in data])
contrastRight = contrastLeft.copy()
contrastLeft[contrastLeft > 0] = np.nan
contrastLeft = np.abs(contrastLeft)
contrastRight[contrastRight < 0] = np.nan
# save if needed
check_alf_folder(session_path)
if raw.save_bool(save, '_ibl_trials.contrastLeft.npy'):
lpath = os.path.join(session_path, 'alf',
'_ibl_trials.contrastLeft.npy')
np.save(lpath, contrastLeft)
if raw.save_bool(save, '_ibl_trials.contrastRight.npy'):
rpath = os.path.join(session_path, 'alf',
'_ibl_trials.contrastRight.npy')
np.save(rpath, contrastRight)
return (contrastLeft, contrastRight)
def get_choice(session_path, save=False, data=False):
"""
Get the subject's choice in every trial.
**Optional:** saves _ibl_trials.choice.npy to alf folder.
Uses signed_contrast and trial_correct.
-1 is a CCW turn (towards the left)
+1 is a CW turn (towards the right)
0 is a no_go trial
If a trial is correct the choice of the animal was the inverse of the sign
of the contrast.
>>> choice[t] = -np.sign(signed_contrast[t]) if trial_correct[t]
:param session_path: absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:type save: bool, optional
:return: numpy.ndarray
:rtype: dtype('int64')
"""
if not data:
data = raw.load_data(session_path)
sitm_side = np.array([np.sign(t['signed_contrast']) for t in data])
trial_correct = np.array([t['trial_correct'] for t in data])
choice = sitm_side.copy()
choice[trial_correct] = -choice[trial_correct]
choice = choice.astype(int)
if raw.save_bool(save, '_ibl_trials.choice.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.choice.npy')
np.save(fpath, choice)
return choice
def from_path(session_path, force=False, save=True):
"""
Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001')
:param force: (False) overwrite existing files
:param save: (True) boolean or list of ALF file names to extract
:return: None
"""
extractor_type = get_session_extractor_type(session_path)
logger_.info(f"Extracting {session_path} as {extractor_type}")
if is_extracted(session_path) and not force:
logger_.info(f"Session {session_path} already extracted.")
return
if extractor_type == 'training':
settings, data = rawio.load_bpod(session_path)
logger_.info('training session on ' + settings['PYBPOD_BOARD'])
training_trials.extract_all(session_path, data=data, save=save)
training_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'biased':
settings, data = rawio.load_bpod(session_path)
logger_.info('biased session on ' + settings['PYBPOD_BOARD'])
biased_trials.extract_all(session_path, data=data, save=save)
biased_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'ephys':
data = rawio.load_data(session_path)
logger_.info('extract BPOD for ephys session')
ephys_trials.extract_all(session_path, data=data, save=save)
logger_.info('extract FPGA information for ephys session')
tmax = data[-1]['behavior_data']['States timestamps']['exit_state'][0][-1] + 60
ephys_fpga.extract_all(session_path, save=save, tmax=tmax)
if extractor_type == 'sync_ephys':
ephys_fpga.extract_sync(session_path, save=save)
def get_goCueOnset_times(session_path, save=False, data=False):
"""
Get trigger times of goCue from state machine.
Current software solution for triggering sounds uses PyBpod soft codes.
Delays can be in the order of 10's of ms. This is the time when the command
to play the sound was executed. To measure accurate time, either getting the
sound onset from the future microphone OR the new xonar soundcard and
setup developed by Sanworks guarantees a set latency (in testing).
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
goCue = np.array([
tr['behavior_data']['States timestamps']['closed_loop'][0][0]
for tr in data
])
if raw.save_bool(save, '_ibl_trials.goCue_times.npy'):
check_alf_folder(session_path)
fpath = Path(session_path).joinpath('alf',
'_ibl_trials.goCue_times.npy')
np.save(fpath, goCue)
return goCue
def check_data(self):
log.info("Checking...")
data = raw.load_data(self.SESSION_FOLDER)
for t in data:
bad_tone_state = t['behavior_data']['States timestamps'][
'bad_tone']
bad_stim_state = t['behavior_data']['States timestamps'][
'bad_stim']
if not np.all(np.isnan(bad_stim_state)):
self.bad_stim_count += 1
if not np.all(np.isnan(bad_tone_state)):
self.bad_tone_count += 1
if (self.bad_stim_count != 0) or (self.bad_tone_count != 0):
log.error(f"""
##########################################
TEST FAILED !!!!!
##########################################
Missing stimulus sync pulses: {self.bad_stim_count}
Missing tone sync pulses: {self.bad_tone_count}
##########################################""")
else:
log.info(f"""
##########################################
TEST PASSED:
##########################################
Stimulus and tone detected in all trials
##########################################""")
def extract_all(session_path, save=False, bpod_trials=None, settings=None):
if not bpod_trials:
bpod_trials = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
base = [
FeedbackType, ContrastLR, ProbabilityLeft, Choice, RepNum,
RewardVolume, FeedbackTimes, Intervals, ResponseTimes,
GoCueTriggerTimes, GoCueTimes
]
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
base.extend([
StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes,
StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes
])
else:
base.extend([IncludedTrials, ItiDuration, StimOnTimes])
out, fil = run_extractor_classes(base,
save=save,
session_path=session_path,
bpod_trials=bpod_trials,
settings=settings)
return out, fil
def get_feedback_times(session_path, save=False, data=False, settings=False):
"""
Get the times the water or error tone was delivered to the animal.
**Optional:** saves _ibl_trials.feedback_times.npy
Gets reward and error state init times vectors,
checks if theintersection of nans is empty, then
merges the 2 vectors.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
merge = get_feedback_times_ge5(session_path, data=data)
else:
merge = get_feedback_times_lt5(session_path, data=data)
if raw.save_bool(save, '_ibl_trials.feedback_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.feedback_times.npy')
np.save(fpath, merge)
return np.array(merge)
def from_path(session_path, force=False, save=True):
"""
Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001')
:param force: (False) overwrite existing files
:param save: (True) boolean or list of ALF file names to extract
:return: None
"""
logger_.info('Extracting ' + str(session_path))
extractor_type = extractors_exist(session_path)
if is_extracted(session_path) and not force:
logger_.info(f"Session {session_path} already extracted.")
return
if extractor_type == 'training':
settings, data = raw.load_bpod(session_path)
logger_.info('training session on ' + settings['PYBPOD_BOARD'])
training_trials.extract_all(session_path, data=data, save=save)
training_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'biased':
settings, data = raw.load_bpod(session_path)
logger_.info('biased session on ' + settings['PYBPOD_BOARD'])
biased_trials.extract_all(session_path, data=data, save=save)
biased_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'ephys':
data = raw.load_data(session_path)
ephys_trials.extract_all(session_path, data=data, save=save)
ephys_fpga.extract_all(session_path, save=save)
def get_feedback_times_ge5(session_path, data=False):
# ger err and no go trig times -- look for BNC2High of trial -- verify
# only 2 onset times go tone and noise, select 2nd/-1 OR select the one
# that is grater than the nogo or err trial onset time
if not data:
data = raw.load_data(session_path)
missed_bnc2 = 0
rw_times, err_sound_times, merge = [
np.zeros([
len(data),
]) for _ in range(3)
]
for ind, tr in enumerate(data):
st = tr['behavior_data']['Events timestamps'].get('BNC2High', None)
if not st:
st = np.array([np.nan, np.nan])
missed_bnc2 += 1
# xonar soundcard duplicates events, remove consecutive events too close together
st = np.delete(st, np.where(np.diff(st) < 0.020)[0] + 1)
rw_times[ind] = tr['behavior_data']['States timestamps']['reward'][
0][0]
# get the error sound only if the reward is nan
err_sound_times[ind] = st[-1] if st.size >= 2 and np.isnan(
rw_times[ind]) else np.nan
if missed_bnc2 == len(data):
_logger.warning(
'No BNC2 for feedback times, filling error trials NaNs')
merge *= np.nan
merge[~np.isnan(rw_times)] = rw_times[~np.isnan(rw_times)]
merge[~np.isnan(err_sound_times
)] = err_sound_times[~np.isnan(err_sound_times)]
return merge
def extract_all(session_path, save=False, bpod_trials=False, settings=False):
"""Extract all datasets from habituationChoiceWorld
Note: only the datasets from the HabituationTrials extractor will be saved to disc.
:param session_path: The session path where the raw data are saved
:param save: If True, the datasets that are considered standard are saved to the session path
:param bpod_trials: The raw Bpod trial data
:param settings: The raw Bpod sessions
:returns: a dict of datasets and a corresponding list of file names
"""
if not bpod_trials:
bpod_trials = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
# Standard datasets that may be saved as ALFs
params = dict(session_path=session_path,
bpod_trials=bpod_trials,
settings=settings)
out, fil = run_extractor_classes(HabituationTrials, save=save, **params)
# The extra datasets
non_standard = [
ItiInTimes, StimOffTriggerTimes, StimCenterTriggerTimes,
StimCenterTimes
]
data, _ = run_extractor_classes(non_standard, save=False, **params)
# Merge the extracted data
out.update(data)
fil.extend([None for _ in data.keys()])
return out, fil
def extract_all(session_path, save=False, tmax=None, bin_exists=True):
"""
For the IBL ephys task, reads ephys binary file and extract:
- sync
- wheel
- behaviour
- video time stamps
:param session_path: '/path/to/subject/yyyy-mm-dd/001'
:param save: Bool, defaults to False
:param version: bpod version, defaults to None
:return: None
"""
session_path = Path(session_path)
alf_path = session_path / 'alf'
if tmax is None:
try:
raw_trials = raw.load_data(session_path)
tmax = raw_trials[-1]['behavior_data']['States timestamps'][
'exit_state'][0][-1] + 60
except Exception:
tmax = np.inf
sync, sync_chmap = _get_main_probe_sync(session_path,
bin_exists=bin_exists)
extract_wheel_sync(sync, alf_path, save=save, chmap=sync_chmap)
extract_camera_sync(sync, alf_path, save=save, chmap=sync_chmap)
extract_behaviour_sync(sync,
alf_path,
save=save,
chmap=sync_chmap,
tmax=tmax)
align_with_bpod(
session_path) # checks consistency and compute dt with bpod
def get_goCueOnset_times(session_path, save=False, data=False, settings=False):
"""
Get trigger times of goCue from state machine.
Current software solution for triggering sounds uses PyBpod soft codes.
Delays can be in the order of 10's of ms. This is the time when the command
to play the sound was executed. To measure accurate time, either getting the
sound onset from the future microphone OR the new xonar soundcard and
setup developed by Sanworks guarantees a set latency (in testing).
:param session_path: Absolute path of session folder
:type session_path: str
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
go_cue_times = np.zeros([
len(data),
])
for ind, tr in enumerate(data):
if get_port_events(tr, 'BNC2'):
bnchigh = tr['behavior_data']['Events timestamps'].get(
'BNC2High', None)
if bnchigh:
go_cue_times[ind] = bnchigh[0]
continue
bnclow = tr['behavior_data']['Events timestamps'].get(
'BNC2Low', None)
if bnclow:
go_cue_times[ind] = bnclow[0] - 0.1
continue
go_cue_times[ind] = np.nan
else:
go_cue_times[ind] = np.nan
nmissing = np.sum(np.isnan(go_cue_times))
# Check if all stim_syncs have failed to be detected
if np.all(np.isnan(go_cue_times)):
logger_.error(
f'{session_path}: Missing ALL BNC2 stimulus ({nmissing} trials')
# Check if any stim_sync has failed be detected for every trial
if np.any(np.isnan(go_cue_times)):
logger_.warning(
f'{session_path}: Missing BNC2 stimulus on {nmissing} trials')
if raw.save_bool(save, '_ibl_trials.goCue_times.npy'):
check_alf_folder(session_path)
fpath = Path(session_path).joinpath('alf',
'_ibl_trials.goCue_times.npy')
np.save(fpath, go_cue_times)
return go_cue_times
def extract_sound(ses_path, save=True, force=False, delete=False):
"""
Simple audio features extraction for ambient sound characterization.
From a wav file, generates several ALF files to be registered on Alyx
:param ses_path: ALF full session path: (/mysubject001/YYYY-MM-DD/001)
:param delete: if True, removes the wav file after processing
:return: list of output files
"""
ses_path = Path(ses_path)
wav_file = ses_path / 'raw_behavior_data' / '_iblrig_micData.raw.wav'
out_folder = ses_path / 'raw_behavior_data'
files_out = {
'power': out_folder / '_iblmic_audioSpectrogram.power.npy',
'frequencies': out_folder / '_iblmic_audioSpectrogram.frequencies.npy',
'onset_times': out_folder / '_iblmic_audioOnsetGoCue.times_mic.npy',
'times_microphone':
out_folder / '_iblmic_audioSpectrogram.times_mic.npy',
}
if not wav_file.exists():
logger_.warning(f"Wav file doesn't exist: {wav_file}")
return [files_out[k] for k in files_out if files_out[k].exists()]
# crunch the wav file
fs, wav = wavfile.read(wav_file, mmap=False)
if len(wav) == 0:
status = _fix_wav_file(wav_file)
if status != 0:
logger_.error(
f"WAV Header empty. Sox couldn't fix it, Abort. {wav_file}")
return
else:
fs, wav = wavfile.read(wav_file, mmap=False)
tscale, fscale, W, detect = welchogram(fs, wav)
# save files
if save:
out_folder.mkdir(exist_ok=True)
np.save(file=files_out['power'], arr=W.astype(np.single))
np.save(file=files_out['frequencies'],
arr=fscale[None, :].astype(np.single))
np.save(file=files_out['onset_times'], arr=detect)
np.save(file=files_out['times_microphone'],
arr=tscale[:, None].astype(np.single))
# for the time scale, attempt to synchronize using onset sound detection and task data
data = ioraw.load_data(ses_path)
if data is None: # if no session data, we're done
if delete:
wav_file.unlink()
return
tgocue, _ = GoCueTimes(ses_path).extract(save=False, bpod_trials=data)
ilast = min(len(tgocue), len(detect))
dt = tgocue[:ilast] - detect[:ilast]
# only save if dt is consistent for the whole session
if np.std(dt) < 0.2 and save:
files_out['times'] = out_folder / '_iblmic_audioSpectrogram.times.npy'
tscale += np.median(dt)
np.save(file=files_out['times'], arr=tscale[:, None].astype(np.single))
if delete:
wav_file.unlink()
return [files_out[k] for k in files_out]
def get_trial_start_times(session_path, data=None):
if not data:
data = raw.load_data(session_path)
trial_start_times = []
for tr in data:
trial_start_times.extend(
[x[0] for x in tr['behavior_data']['States timestamps']['trial_start']])
return np.array(trial_start_times)
def get_laser(session_path, save=False, data=False):
if not data:
data = raw.load_data(session_path)
laser_on = np.array([t['laser_on'] for t in data])
if raw.save_bool(save, '_ibl_trials.laseron.npy'):
lpath = Path(session_path).joinpath('alf', '_ibl_trials.laseron.npy')
np.save(lpath, laser_on)
return laser_on
def test_get_choice(self):
# TRAINING SESSIONS
choice = extractors.training_trials.Choice(
session_path=self.training_lt5['path']).extract(save=False)[0]
self.assertTrue(isinstance(choice, np.ndarray))
data = raw.load_data(self.training_lt5['path'])
trial_nogo = np.array([
~np.isnan(t['behavior_data']['States timestamps']['no_go'][0][0])
for t in data
])
if any(trial_nogo):
self.assertTrue(all(choice[trial_nogo]) == 0)
# -- version >= 5.0.0
choice = extractors.training_trials.Choice(
session_path=self.training_ge5['path']).extract(save=False)[0]
self.assertTrue(isinstance(choice, np.ndarray))
data = raw.load_data(self.training_ge5['path'])
trial_nogo = np.array([
~np.isnan(t['behavior_data']['States timestamps']['no_go'][0][0])
for t in data
])
if any(trial_nogo):
self.assertTrue(all(choice[trial_nogo]) == 0)
# BIASED SESSIONS
choice = extractors.biased_trials.Choice(
session_path=self.biased_lt5['path']).extract(save=False)[0]
self.assertTrue(isinstance(choice, np.ndarray))
data = raw.load_data(self.biased_lt5['path'])
trial_nogo = np.array([
~np.isnan(t['behavior_data']['States timestamps']['no_go'][0][0])
for t in data
])
if any(trial_nogo):
self.assertTrue(all(choice[trial_nogo]) == 0)
# -- version >= 5.0.0
choice = extractors.biased_trials.Choice(
session_path=self.biased_ge5['path']).extract(save=False)[0]
self.assertTrue(isinstance(choice, np.ndarray))
data = raw.load_data(self.biased_ge5['path'])
trial_nogo = np.array([
~np.isnan(t['behavior_data']['States timestamps']['no_go'][0][0])
for t in data
])
if any(trial_nogo):
self.assertTrue(all(choice[trial_nogo]) == 0)
