def test_load_embedded_frame_data(self):
session = Path(__file__).parent.joinpath('extractors', 'data', 'session_ephys')
count, gpio = raw.load_embedded_frame_data(session, 'body')
self.assertEqual(count[0], 0)
self.assertIsInstance(gpio[-1], dict)
count, gpio = raw.load_embedded_frame_data(session, 'body', raw=True)
self.assertNotEqual(count[0], 0)
self.assertIsInstance(gpio, np.ndarray)
def test_groom_pin_state(self):
# ibl_witten_27\2021-01-14\001 # Can't assign a pin state
# CSK-im-002\2021-01-16\001 # Another example
root = self.data_path
session_path = root.joinpath('ephys', 'ephys_choice_world_task',
'ibl_witten_27', '2021-01-21', '001')
_, ts = raw.load_camera_ssv_times(session_path, 'left')
_, (*_, gpio) = raw.load_embedded_frame_data(session_path, 'left')
bpod_trials = raw.load_data(session_path)
_, audio = raw.load_bpod_fronts(session_path, bpod_trials)
# NB: syncing the timestamps to the audio doesn't work very well but we don't need it to
# for the extraction, so long as the audio and GPIO fronts match.
gpio, audio, _ = camio.groom_pin_state(gpio, audio, ts)
# Do some checks
self.assertEqual(gpio['indices'].size, audio['times'].size)
expected = np.array([164179, 164391, 164397, 164900, 164906], dtype=int)
np.testing.assert_array_equal(gpio['indices'][-5:], expected)
expected = np.array([2734.4496, 2737.9659, 2738.0659, 2746.4488, 2746.5488])
np.testing.assert_array_almost_equal(audio['times'][-5:], expected)
# Verify behaviour when audio and GPIO match in size
_, audio_, _ = camio.groom_pin_state(gpio, audio, ts, take='nearest', tolerance=.5)
self.assertEqual(audio, audio_)
# Verify behaviour when there are GPIO fronts beyond number of video frames
ts_short = ts[:gpio['indices'].max() - 10]
gpio_, *_ = camio.groom_pin_state(gpio, audio, ts_short)
self.assertFalse(np.any(gpio_['indices'] >= ts.size))
def test_groom_pin_state(self):
"""
e7098000-62a0-46a4-99df-981ee2b56988 (ZFM-01867/2/2021-03-23)
In this session there were occasions where the GPIO would change twice after
an audio TTL, perhaps because the audio TTLs are often split up into two short
TTLs on Bpod, some of which are caught by the camera, others not.
The function removes the short audio TTLs then assigns the rest to the GPIO fronts.
The unassigned audio TTL fronts and GPIO changes are removed. Usually if the TTL
low-to-high is not assigned to a GPIO front, neither is the high-to-low, so both are
removed. However sometimes because of mis-assigning (due to clock drift, short TTLs,
faulty wiring, etc.) there are some 'orphaned' TTLs/GPIO fronts leaving us with two
low-to-high fronts (or high-to-low) in a row. These so-called orphaned fronts
should be removed too. The goal is to end up with an array of audio TTL times and
GPIO times that are the same length.
Debugging output states:
- 2316 fronts TLLs less than 5ms apart
- 11 audio TTL rises were not detected by the camera
- 346 pin state rises could not be attributed to an audio TTL
- 10 audio TTL falls were not detected by the camera
- 345 pin state falls could not be attributed to an audio TTL
- 3 orphaned TTLs removed
The output arrays are not aligned per se, but should at least have *most* GPIO fronts
correctly assigned to the corresponding audio TTLs.
:return:
"""
root = self.data_path
session_path = root.joinpath('camera', 'ZFM-01867', '2021-03-23', '002')
_, ts = raw.load_camera_ssv_times(session_path, 'left')
_, (*_, gpio) = raw.load_embedded_frame_data(session_path, 'left')
bpod_trials = raw.load_data(session_path)
_, audio = raw.load_bpod_fronts(session_path, bpod_trials)
# NB: syncing the timestamps to the audio doesn't work very well but we don't need it to
# for the extraction, so long as the audio and GPIO fronts match.
gpio, audio, _ = camio.groom_pin_state(gpio, audio, ts,
take='nearest', tolerance=.5, min_diff=5e-3)
# Do some checks
self.assertEqual(gpio['indices'].size, audio['times'].size)
expected = np.array([446328, 446812, 446814, 447251, 447253], dtype=int)
np.testing.assert_array_equal(gpio['indices'][-5:], expected)
expected = np.array([4448.100798, 4452.912398, 4452.934398, 4457.313998, 4457.335998])
np.testing.assert_array_almost_equal(audio['times'][-5:], expected)
def _extract(self,
sync=None,
chmap=None,
video_path=None,
display=False,
extrapolate_missing=True):
"""
The raw timestamps are taken from the FPGA. These are the times of the camera's frame TTLs.
If the pin state file exists, these timestamps are aligned to the video frames using the
audio TTLs. Frames missing from the embedded frame count are removed from the timestamps
array.
If the pin state file does not exist, the left and right camera timestamps may be aligned
using the wheel data.
:param sync: dictionary 'times', 'polarities' of fronts detected on sync trace.
:param chmap: dictionary containing channel indices. Default to constant.
:param video_path: an optional path for fetching the number of frames. If None,
the video is loaded from the session path. If an int is provided this is taken to be
the total number of frames.
:param display: if True, the audio and GPIO fronts are plotted.
:param extrapolate_missing: if True, any missing timestamps at the beginning and end of
the session are extrapolated based on the median frame rate, otherwise they will be NaNs.
:return: a numpy array of camera timestamps
"""
fpga_times = extract_camera_sync(sync=sync, chmap=chmap)
count, (*_,
gpio) = raw.load_embedded_frame_data(self.session_path,
self.label)
raw_ts = fpga_times[self.label]
if video_path is None:
filename = f'_iblrig_{self.label}Camera.raw.mp4'
video_path = self.session_path.joinpath('raw_video_data', filename)
# Permit the video path to be the length for development and debugging purposes
length = (video_path if isinstance(video_path, int) else
get_video_length(video_path))
_logger.debug(f'Number of video frames = {length}')
if gpio is not None and gpio['indices'].size > 1:
_logger.info('Aligning to audio TTLs')
# Extract audio TTLs
audio = get_sync_fronts(sync, chmap['audio'])
_, ts = raw.load_camera_ssv_times(self.session_path, self.label)
try:
"""
NB: Some of the audio TTLs occur very close together, and are therefore not
reflected in the pin state. This function removes those. Also converts frame
times to FPGA time.
"""
gpio, audio, ts = groom_pin_state(gpio,
audio,
ts,
display=display)
"""
The length of the count and pin state are regularly longer than the length of
the video file. Here we assert that the video is either shorter or the same
length as the arrays, and we make an assumption that the missing frames are
right at the end of the video. We therefore simply shorten the arrays to match
the length of the video.
"""
if count.size > length:
count = count[:length]
else:
assert length == count.size, 'fewer counts than frames'
raw_ts = fpga_times[self.label]
assert raw_ts.shape[0] > 0, 'no timestamps found in channel indicated for ' \
f'{self.label} camera'
return align_with_audio(
raw_ts,
audio,
gpio,
count,
display=display,
extrapolate_missing=extrapolate_missing)
except AssertionError as ex:
_logger.critical('Failed to extract using audio: %s', ex)
# If you reach here extracting using audio TTLs was not possible
_logger.warning('Alignment by wheel data not yet implemented')
if length < raw_ts.size:
df = raw_ts.size - length
_logger.info(f'Discarding first {df} pulses')
raw_ts = raw_ts[df:]
return raw_ts
def _extract(self,
video_path=None,
display=False,
extrapolate_missing=True):
"""
The raw timestamps are taken from the Bpod. These are the times of the camera's frame TTLs.
If the pin state file exists, these timestamps are aligned to the video frames using the
audio TTLs. Frames missing from the embedded frame count are removed from the timestamps
array.
If the pin state file does not exist, the left camera timestamps may be aligned using the
wheel data.
:param video_path: an optional path for fetching the number of frames. If None,
the video is loaded from the session path. If an int is provided this is taken to be
the total number of frames.
:param display: if True, the audio and GPIO fronts are plotted.
:param extrapolate_missing: if True, any missing timestamps at the beginning and end of
the session are extrapolated based on the median frame rate, otherwise they will be NaNs.
:return: a numpy array of camera timestamps
"""
raw_ts = self._times_from_bpod()
count, (*_,
gpio) = raw.load_embedded_frame_data(self.session_path, 'left')
if video_path is None:
filename = '_iblrig_leftCamera.raw.mp4'
video_path = self.session_path.joinpath('raw_video_data', filename)
# Permit the video path to be the length for development and debugging purposes
length = video_path if isinstance(
video_path, int) else get_video_length(video_path)
_logger.debug(f'Number of video frames = {length}')
# Check if the GPIO is usable for extraction. GPIO is None if the file does not exist,
# is empty, or contains only one value (i.e. doesn't change)
if gpio is not None and gpio['indices'].size > 1:
_logger.info('Aligning to audio TTLs')
# Extract audio TTLs
_, audio = raw.load_bpod_fronts(self.session_path,
self.bpod_trials)
_, ts = raw.load_camera_ssv_times(self.session_path, 'left')
"""
There are many audio TTLs that are for some reason missed by the GPIO. Conversely
the last GPIO doesn't often correspond to any audio TTL. These will be removed.
The drift appears to be less severe than the FPGA, so when assigning TTLs we'll take
the nearest TTL within 500ms. The go cue TTLs comprise two short pulses ~3ms apart.
We will fuse any TTLs less than 5ms apart to make assignment more accurate.
"""
try:
gpio, audio, ts = groom_pin_state(gpio,
audio,
ts,
take='nearest',
tolerance=.5,
min_diff=5e-3,
display=display)
if count.size > length:
count = count[:length]
else:
assert length == count.size, 'fewer counts than frames'
return align_with_audio(raw_ts,
audio,
gpio,
count,
extrapolate_missing,
display=display)
except AssertionError as ex:
_logger.critical('Failed to extract using audio: %s', ex)
# If you reach here extracting using audio TTLs was not possible
_logger.warning('Alignment by wheel data not yet implemented')
# Extrapolate at median frame rate
n_missing = length - raw_ts.size
if n_missing > 0:
_logger.warning(
f'{n_missing} fewer Bpod timestamps than frames; '
f'{"extrapolating" if extrapolate_missing else "appending nans"}'
)
frate = np.median(np.diff(raw_ts))
to_app = ((np.arange(n_missing, ) + 1) / frate + raw_ts[-1]
if extrapolate_missing else np.full(n_missing, np.nan))
raw_ts = np.r_[raw_ts, to_app] # Append the missing times
elif n_missing < 0:
_logger.warning(
f'{abs(n_missing)} fewer frames than Bpod timestamps')
_logger.info(f'Discarding first {abs(n_missing)} pulses')
raw_ts = raw_ts[abs(n_missing):]
return raw_ts
def load_data(self,
download_data: bool = None,
extract_times: bool = False,
load_video: bool = True) -> None:
"""Extract the data from raw data files
Extracts all the required task data from the raw data files.
Data keys:
- count (int array): the sequential frame number (n, n+1, n+2...)
- pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame
- audio (float array): timestamps of audio TTL fronts
- fpga_times (float array): timestamps of camera TTLs recorded by FPGA
- timestamps (float array): extracted video timestamps (the camera.times ALF)
- bonsai_times (datetime array): system timestamps of video PC; should be one per frame
- camera_times (float array): camera frame timestamps extracted from frame headers
- wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion
- video (Bunch): video meta data, including dimensions and FPS
- frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel)
:param download_data: if True, any missing raw data is downloaded via ONE.
Missing data will raise an AssertionError
:param extract_times: if True, the camera.times are re-extracted from the raw data
:param load_video: if True, calls the load_video_data method
"""
assert self.session_path, 'no session path set'
if download_data is not None:
self.download_data = download_data
if self.download_data and self.eid and self.one and not self.one.offline:
self.ensure_required_data()
_log.info('Gathering data for QC')
# Get frame count and pin state
self.data['count'], self.data['pin_state'] = \
raw.load_embedded_frame_data(self.session_path, self.label, raw=True)
# Load the audio and raw FPGA times
if self.type == 'ephys':
sync, chmap = ephys_fpga.get_main_probe_sync(self.session_path)
audio_ttls = ephys_fpga.get_sync_fronts(sync, chmap['audio'])
self.data['audio'] = audio_ttls['times'] # Get rises
# Load raw FPGA times
cam_ts = extract_camera_sync(sync, chmap)
self.data['fpga_times'] = cam_ts[self.label]
else:
bpod_data = raw.load_data(self.session_path)
_, audio_ttls = raw.load_bpod_fronts(self.session_path, bpod_data)
self.data['audio'] = audio_ttls['times']
# Load extracted frame times
alf_path = self.session_path / 'alf'
try:
assert not extract_times
self.data['timestamps'] = alfio.load_object(
alf_path, f'{self.label}Camera', short_keys=True)['times']
except AssertionError: # Re-extract
kwargs = dict(video_path=self.video_path, labels=self.label)
if self.type == 'ephys':
kwargs = {**kwargs, 'sync': sync, 'chmap': chmap} # noqa
outputs, _ = extract_all(self.session_path,
self.type,
save=False,
**kwargs)
self.data['timestamps'] = outputs[
f'{self.label}_camera_timestamps']
except ALFObjectNotFound:
_log.warning('no camera.times ALF found for session')
# Get audio and wheel data
wheel_keys = ('timestamps', 'position')
try:
self.data['wheel'] = alfio.load_object(alf_path,
'wheel',
short_keys=True)
except ALFObjectNotFound:
# Extract from raw data
if self.type == 'ephys':
wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap)
else:
wheel_data = training_wheel.get_wheel_position(
self.session_path)
self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data))
# Find short period of wheel motion for motion correlation.
if data_for_keys(
wheel_keys,
self.data['wheel']) and self.data['timestamps'] is not None:
self.data['wheel'].period = self.get_active_wheel_period(
self.data['wheel'])
# Load Bonsai frame timestamps
try:
ssv_times = raw.load_camera_ssv_times(self.session_path,
self.label)
self.data['bonsai_times'], self.data['camera_times'] = ssv_times
except AssertionError:
_log.warning('No Bonsai video timestamps file found')
# Gather information from video file
if load_video:
_log.info('Inspecting video file...')
self.load_video_data()