def test_load_encoder_positions(self):
raw.load_encoder_positions(self.training_lt5['path'],
settings={'IBLRIG_VERSION_TAG': '4.9.9'})
raw.load_encoder_positions(self.training_ge5['path'])
raw.load_encoder_positions(self.biased_lt5['path'],
settings={'IBLRIG_VERSION_TAG': '4.9.9'})
raw.load_encoder_positions(self.biased_ge5['path'])
def test_encoder_positions_clock_reset(self):
dy = loaders.load_encoder_positions(self.session_path)
dat = np.array([
849736, 1532230, 1822449, 1833514, 1841566, 1848206, 1853979,
1859144
])
self.assertTrue(np.all(np.diff(dy['re_ts']) > 0))
self.assertTrue(all(dy['re_ts'][6:] - 2**32 - dat == 0))
def get_wheel_data(session_path, save=False):
"""
Get wheel data from raw files and converts positions into centimeters and
timestamps into seconds.
**Optional:** saves _ibl_wheel.times.npy and _ibl_wheel.position.npy
Times:
Gets Rotary Encoder timestamps (ms) for each position and converts to times.
Uses time_converter to extract and convert timstamps (ms) to times (s).
Positions:
Positions are in (cm) of RE perimeter relative to 0. The 0 resets every trial.
cmtick = radius (cm) * 2 * pi / n_ticks
cmtick = 3.1 * 2 * np.pi / 1024
:param session_path: absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:type save: bool, optional
:return: Numpy structured array.
:rtype: numpy.ndarray
"""
df = raw.load_encoder_positions(session_path)
names = df.columns.tolist()
data = structarr(names, shape=(df.index.max() + 1, ))
data['re_ts'] = df.re_ts.values
data['re_pos'] = df.re_pos.values
data['bns_ts'] = df.bns_ts.values
# ticks to cm factor
cmtick = 3.1 * 2 * np.pi / 1024
# Convert position and timestamps to cm and seconds respectively
data['re_ts'] = data['re_ts'] / 1000.
data['re_pos'] = data['re_pos'] * cmtick
# Find timestamps that are repeated
rep_idx = np.where(np.diff(data['re_ts']) == 0)[0]
# Change the value of the repeated position
data['re_pos'][rep_idx] = (data['re_pos'][rep_idx] +
data['re_pos'][rep_idx + 1]) / 2
# get the converter function to translate re_ts into behavior times
convtime = time_converter_session(session_path, kind='re2b')
data['re_ts'] = convtime(data['re_ts'])
# Now remove the repeted times that are rep_idx + 1
data = np.delete(data, rep_idx + 1)
check_alf_folder(session_path)
if raw.save_bool(save, '_ibl_wheel.timestamps.npy'):
tpath = os.path.join(session_path, 'alf', '_ibl_wheel.timestamps.npy')
np.save(tpath, data['re_ts'])
if raw.save_bool(save, '_ibl_wheel.position.npy'):
ppath = os.path.join(session_path, 'alf', '_ibl_wheel.position.npy')
np.save(ppath, data['re_pos'])
return data
def get_wheel_data(session_path, bp_data=None, save=False):
"""
Get wheel data from raw files and converts positions into centimeters and
timestamps into seconds.
**Optional:** saves _ibl_wheel.times.npy and _ibl_wheel.position.npy
Times:
Gets Rotary Encoder timestamps (ms) for each position and converts to times.
Uses time_converter to extract and convert timstamps (ms) to times (s).
Positions:
Positions are in (cm) of RE perimeter relative to 0. The 0 resets every trial.
cmtick = radius (cm) * 2 * pi / n_ticks
cmtick = 3.1 * 2 * np.pi / 1024
:param session_path: absolute path of session folder
:type session_path: str
:param data: dictionary containing the contents pybppod jsonable file read with raw.load_data
:type data: dict, optional
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:type save: bool, optional
:return: Numpy structured array.
:rtype: numpy.ndarray
"""
##
status = 0
if not bp_data:
bp_data = raw.load_data(session_path)
df = raw.load_encoder_positions(session_path)
if df is None:
logger_.error('No wheel data for ' + str(session_path))
return None
data = structarr(['re_ts', 're_pos', 'bns_ts'],
shape=(df.shape[0],), formats=['f8', 'f8', np.object])
data['re_ts'] = df.re_ts.values
data['re_pos'] = df.re_pos.values
data['re_pos'] = data['re_pos'] / 1024 * 2 * np.pi # convert positions to radians
trial_starts = get_trial_start_times(session_path)
# need a flag if the data resolution is 1ms due to the old version of rotary encoder firmware
if np.all(np.mod(data['re_ts'], 1e3) == 0):
status = 1
data['re_ts'] = data['re_ts'] / 1e6 # convert ts to seconds
# get the converter function to translate re_ts into behavior times
convtime = time_converter_session(session_path, kind='re2b')
data['re_ts'] = convtime(data['re_ts'])
return data
def get_reset_trace_compensation_with_state_machine_times():
# this is the preferred way of getting resets using the state machine time information
# it will not always work depending on firmware versions, new bugs
iwarn = []
ns = len(data['re_pos'])
tr_dc = np.zeros_like(data['re_pos']) # trial dc component
for bp_dat in bp_data:
restarts = np.sort(np.array(
bp_dat['behavior_data']['States timestamps']['reset_rotary_encoder'] +
bp_dat['behavior_data']['States timestamps']['reset2_rotary_encoder'])[:, 0])
ind = np.unique(np.searchsorted(data['re_ts'], restarts, side='left') - 1)
# the rotary encoder doesn't always reset right away, and the reset sample given the
# timestamp can be ambiguous: look for zeros
for i in np.where(data['re_pos'][ind] != 0)[0]:
# handle boundary effects
if ind[i] > ns - 2:
continue
# it happens quite often that we have to lock in to next sample to find the reset
if data['re_pos'][ind[i] + 1] == 0:
ind[i] = ind[i] + 1
continue
# also case where the rotary doesn't reset to 0, but erratically to -1/+1
if data['re_pos'][ind[i]] <= (1 / 1024 * 2 * np.pi):
ind[i] = ind[i] + 1
continue
# compounded with the fact that the reset may have happened at next sample.
if np.abs(data['re_pos'][ind[i] + 1]) <= (1 / 1024 * 2 * np.pi):
ind[i] = ind[i] + 1
continue
# sometimes it is also the last trial that has this behaviour
if (bp_data[-1] is bp_dat) or (bp_data[0] is bp_dat):
continue
iwarn.append(ind[i])
# at which point we are running out of possible bugs and calling it
tr_dc[ind] = data['re_pos'][ind - 1]
if iwarn: # if a warning flag was caught in the loop throw a single warning
logger_.warning('Rotary encoder reset events discrepancy. Doing my best to merge.')
logger_.debug('Offending inds: ' + str(iwarn) + ' times: ' + str(data['re_ts'][iwarn]))
# exit status 0 is fine, 1 something went wrong
return tr_dc, len(iwarn) != 0
# attempt to get the resets properly unless the unit is ms which means precision is
# not good enough to match SM times to wheel samples time
if not status:
tr_dc, status = get_reset_trace_compensation_with_state_machine_times()
# if something was wrong or went wrong agnostic way of getting resets: just get zeros values
if status:
tr_dc = np.zeros_like(data['re_pos']) # trial dc component
i0 = np.where(data['re_pos'] == 0)[0]
tr_dc[i0] = data['re_pos'][i0 - 1]
# even if things went ok, rotary encoder may not log the whole session. Need to fix outside
else:
i0 = np.where(np.bitwise_and(np.bitwise_or(data['re_ts'] >= trial_starts[-1],
data['re_ts'] <= trial_starts[0]),
data['re_pos'] == 0))[0]
# make sure the bounds are not included in the current list
i0 = np.delete(i0, np.where(np.bitwise_or(i0 >= len(data['re_pos']) - 1, i0 == 0)))
# a 0 sample is not a reset if 2 conditions are met:
# 1/2 no inflexion (continuous derivative)
c1 = np.abs(np.sign(data['re_pos'][i0 + 1] - data['re_pos'][i0]) -
np.sign(data['re_pos'][i0] - data['re_pos'][i0 - 1])) == 2
# 2/2 needs to be below threshold
c2 = np.abs((data['re_pos'][i0] - data['re_pos'][i0 - 1]) /
(EPS + (data['re_ts'][i0] - data['re_ts'][i0 - 1]))) < THRESHOLD_RAD_PER_SEC
# apply reset to points identified as resets
i0 = i0[np.where(np.bitwise_not(np.bitwise_and(c1, c2)))]
tr_dc[i0] = data['re_pos'][i0 - 1]
# unwrap the rotation (in radians) and then add the DC component from restarts
data['re_pos'] = np.unwrap(data['re_pos']) + np.cumsum(tr_dc)
# Also forgot to mention that time stamps may be repeated or very close to one another.
# Find them as they will induce large jitters on the velocity function or errors in
# attempts of interpolation
rep_idx = np.where(np.diff(data['re_ts']) <= THRESHOLD_CONSECUTIVE_SAMPLES)[0]
# Change the value of the repeated position
data['re_pos'][rep_idx] = (data['re_pos'][rep_idx] +
data['re_pos'][rep_idx + 1]) / 2
data['re_ts'][rep_idx] = (data['re_ts'][rep_idx] +
data['re_ts'][rep_idx + 1]) / 2
# Now remove the repeat times that are rep_idx + 1
data = np.delete(data, rep_idx + 1)
# convert to cm
data['re_pos'] = data['re_pos'] * WHEEL_RADIUS_CM
# # DEBUG PLOTS START HERE ########################
# # if you are experiencing a new bug here is some plot tools
# # do not forget to increment the wasted dev hours counter below
# WASTED_HOURS_ON_THIS_WHEEL_FORMAT = 16
#
# import matplotlib.pyplot as plt
# fig = plt.figure()
# ax = plt.axes()
# tstart = get_trial_start_times(session_path)
# tts = np.c_[tstart, tstart, tstart + np.nan].flatten()
# vts = np.c_[tstart * 0 + 100, tstart * 0 - 100, tstart + np.nan].flatten()
# ax.plot(tts, vts, label='Trial starts')
# ax.plot(convtime(df.re_ts.values/1e6), df.re_pos.values / 1024 * 2 * np.pi,
# '.-', label='Raw data')
# i0 = np.where(df.re_pos.values == 0)
# ax.plot(convtime(df.re_ts.values[i0] / 1e6), df.re_pos.values[i0] / 1024 * 2 * np.pi,
# 'r*', label='Raw data zero samples')
# ax.plot(convtime(df.re_ts.values / 1e6) , tr_dc, label='reset compensation')
# ax.set_xlabel('Bpod Time')
# ax.set_ylabel('radians')
# #
# restarts = np.array(bp_data[10]['behavior_data']['States timestamps']\
# ['reset_rotary_encoder']).flatten()
# # x__ = np.c_[restarts, restarts, restarts + np.nan].flatten()
# # y__ = np.c_[restarts * 0 + 1, restarts * 0 - 1, restarts+ np.nan].flatten()
# #
# # ax.plot(x__, y__, 'k', label='Restarts')
#
# ax.plot(data['re_ts'], data['re_pos'] / WHEEL_RADIUS_CM, '.-', label='Output Trace')
# ax.legend()
# # plt.hist(np.diff(data['re_ts']), 400, range=[0, 0.01])
# # DEBUG PLOTS STOP HERE ########################
check_alf_folder(session_path)
if raw.save_bool(save, '_ibl_wheel.timestamps.npy'):
tpath = os.path.join(session_path, 'alf', '_ibl_wheel.timestamps.npy')
np.save(tpath, data['re_ts'])
if raw.save_bool(save, '_ibl_wheel.position.npy'):
ppath = os.path.join(session_path, 'alf', '_ibl_wheel.position.npy')
np.save(ppath, data['re_pos'])
return data
def test_encoder_positions_clock_errors(self):
# here we test for 2 kinds of file corruption that happen
# 1/2 the first sample time is corrupt and absurdly high and should be discarded
# 2/2 2 samples are swapped and need to be swapped back
dy = loaders.load_encoder_positions(self.session_path_biased)
self.assertTrue(np.all(np.diff(np.array(dy.re_ts)) > 0))
def test_encoder_positions_duds(self):
dy = loaders.load_encoder_positions(self.session_path)
self.assertEqual(dy.bns_ts.dtype.name, 'object')
self.assertTrue(dy.shape[0] == 14)
def get_wheel_position(session_path, bp_data=None, display=False):
"""
Gets wheel timestamps and position from Bpod data. Position is in radian (constant above for
radius is 1) mathematical convention.
:param session_path:
:param bp_data (optional): bpod trials read from jsonable file
:param display (optional): (bool)
:return: timestamps (np.array)
:return: positions (np.array)
"""
status = 0
if not bp_data:
bp_data = raw.load_data(session_path)
df = raw.load_encoder_positions(session_path)
if df is None:
_logger.error('No wheel data for ' + str(session_path))
return None, None
data = structarr(['re_ts', 're_pos', 'bns_ts'],
shape=(df.shape[0],), formats=['f8', 'f8', object])
data['re_ts'] = df.re_ts.values
data['re_pos'] = df.re_pos.values * -1 # anti-clockwise is positive in our output
data['re_pos'] = data['re_pos'] / 1024 * 2 * np.pi # convert positions to radians
trial_starts = get_trial_start_times(session_path)
# need a flag if the data resolution is 1ms due to the old version of rotary encoder firmware
if np.all(np.mod(data['re_ts'], 1e3) == 0):
status = 1
data['re_ts'] = data['re_ts'] / 1e6 # convert ts to seconds
# # get the converter function to translate re_ts into behavior times
re2bpod = sync_rotary_encoder(session_path)
data['re_ts'] = re2bpod(data['re_ts'])
def get_reset_trace_compensation_with_state_machine_times():
# this is the preferred way of getting resets using the state machine time information
# it will not always work depending on firmware versions, new bugs
iwarn = []
ns = len(data['re_pos'])
tr_dc = np.zeros_like(data['re_pos']) # trial dc component
for bp_dat in bp_data:
restarts = np.sort(np.array(
bp_dat['behavior_data']['States timestamps']['reset_rotary_encoder'] +
bp_dat['behavior_data']['States timestamps']['reset2_rotary_encoder'])[:, 0])
ind = np.unique(np.searchsorted(data['re_ts'], restarts, side='left') - 1)
# the rotary encoder doesn't always reset right away, and the reset sample given the
# timestamp can be ambiguous: look for zeros
for i in np.where(data['re_pos'][ind] != 0)[0]:
# handle boundary effects
if ind[i] > ns - 2:
continue
# it happens quite often that we have to lock in to next sample to find the reset
if data['re_pos'][ind[i] + 1] == 0:
ind[i] = ind[i] + 1
continue
# also case where the rotary doesn't reset to 0, but erratically to -1/+1
if data['re_pos'][ind[i]] <= (1 / 1024 * 2 * np.pi):
ind[i] = ind[i] + 1
continue
# compounded with the fact that the reset may have happened at next sample.
if np.abs(data['re_pos'][ind[i] + 1]) <= (1 / 1024 * 2 * np.pi):
ind[i] = ind[i] + 1
continue
# sometimes it is also the last trial that has this behaviour
if (bp_data[-1] is bp_dat) or (bp_data[0] is bp_dat):
continue
iwarn.append(ind[i])
# at which point we are running out of possible bugs and calling it
tr_dc[ind] = data['re_pos'][ind - 1]
if iwarn: # if a warning flag was caught in the loop throw a single warning
_logger.warning('Rotary encoder reset events discrepancy. Doing my best to merge.')
_logger.debug('Offending inds: ' + str(iwarn) + ' times: ' + str(data['re_ts'][iwarn]))
# exit status 0 is fine, 1 something went wrong
return tr_dc, len(iwarn) != 0
# attempt to get the resets properly unless the unit is ms which means precision is
# not good enough to match SM times to wheel samples time
if not status:
tr_dc, status = get_reset_trace_compensation_with_state_machine_times()
# if something was wrong or went wrong agnostic way of getting resets: just get zeros values
if status:
tr_dc = np.zeros_like(data['re_pos']) # trial dc component
i0 = np.where(data['re_pos'] == 0)[0]
tr_dc[i0] = data['re_pos'][i0 - 1]
# even if things went ok, rotary encoder may not log the whole session. Need to fix outside
else:
i0 = np.where(np.bitwise_and(np.bitwise_or(data['re_ts'] >= trial_starts[-1],
data['re_ts'] <= trial_starts[0]),
data['re_pos'] == 0))[0]
# make sure the bounds are not included in the current list
i0 = np.delete(i0, np.where(np.bitwise_or(i0 >= len(data['re_pos']) - 1, i0 == 0)))
# a 0 sample is not a reset if 2 conditions are met:
# 1/2 no inflexion (continuous derivative)
c1 = np.abs(np.sign(data['re_pos'][i0 + 1] - data['re_pos'][i0]) -
np.sign(data['re_pos'][i0] - data['re_pos'][i0 - 1])) == 2
# 2/2 needs to be below threshold
c2 = np.abs((data['re_pos'][i0] - data['re_pos'][i0 - 1]) /
(EPS + (data['re_ts'][i0] - data['re_ts'][i0 - 1]))) < THRESHOLD_RAD_PER_SEC
# apply reset to points identified as resets
i0 = i0[np.where(np.bitwise_not(np.bitwise_and(c1, c2)))]
tr_dc[i0] = data['re_pos'][i0 - 1]
# unwrap the rotation (in radians) and then add the DC component from restarts
data['re_pos'] = np.unwrap(data['re_pos']) + np.cumsum(tr_dc)
# Also forgot to mention that time stamps may be repeated or very close to one another.
# Find them as they will induce large jitters on the velocity function or errors in
# attempts of interpolation
rep_idx = np.where(np.diff(data['re_ts']) <= THRESHOLD_CONSECUTIVE_SAMPLES)[0]
# Change the value of the repeated position
data['re_pos'][rep_idx] = (data['re_pos'][rep_idx] +
data['re_pos'][rep_idx + 1]) / 2
data['re_ts'][rep_idx] = (data['re_ts'][rep_idx] +
data['re_ts'][rep_idx + 1]) / 2
# Now remove the repeat times that are rep_idx + 1
data = np.delete(data, rep_idx + 1)
# convert to cm
data['re_pos'] = data['re_pos'] * WHEEL_RADIUS_CM
# DEBUG PLOTS START HERE ########################
if display:
import matplotlib.pyplot as plt
plt.figure()
ax = plt.axes()
tstart = get_trial_start_times(session_path)
tts = np.c_[tstart, tstart, tstart + np.nan].flatten()
vts = np.c_[tstart * 0 + 100, tstart * 0 - 100, tstart + np.nan].flatten()
ax.plot(tts, vts, label='Trial starts')
ax.plot(re2bpod(df.re_ts.values / 1e6), df.re_pos.values / 1024 * 2 * np.pi,
'.-', label='Raw data')
i0 = np.where(df.re_pos.values == 0)
ax.plot(re2bpod(df.re_ts.values[i0] / 1e6), df.re_pos.values[i0] / 1024 * 2 * np.pi,
'r*', label='Raw data zero samples')
ax.plot(re2bpod(df.re_ts.values / 1e6), tr_dc, label='reset compensation')
ax.set_xlabel('Bpod Time')
ax.set_ylabel('radians')
# restarts = np.array(bp_data[10]['behavior_data']['States timestamps']
# ['reset_rotary_encoder']).flatten()
# x__ = np.c_[restarts, restarts, restarts + np.nan].flatten()
# y__ = np.c_[restarts * 0 + 1, restarts * 0 - 1, restarts+ np.nan].flatten()
# ax.plot(x__, y__, 'k', label='Restarts')
ax.plot(data['re_ts'], data['re_pos'] / WHEEL_RADIUS_CM, '.-', label='Output Trace')
ax.legend()
# plt.hist(np.diff(data['re_ts']), 400, range=[0, 0.01])
return data['re_ts'], data['re_pos']
def test_encoder_positions_duds(self):
dy = loaders.load_encoder_positions(self.session_path)
self.assertEqual(dy.bns_ts.dtype.name, 'datetime64[ns]')
self.assertTrue(dy.shape[0] == 2)
