othread.started_running.wait()
athread = fh.ArduinoThread()
athread.start()
athread.started_running.wait()
probe = fh.FourMarkerProbe()
n_samples = 5
led_positions = np.full((n_samples, 3), np.nan)
rig_calibration_indices = np.round(np.linspace(0, 254, n_samples)).astype(np.int)
for i, ci in enumerate(rig_calibration_indices):
athread.write_uint8(ci, 100, 0, 0)
while True:
fh.wait_for_keypress(pygame.K_SPACE)
probe_rotation, probe_tip = probe.solve(othread.current_sample.copy()[15:27].reshape(4,3))
if fh.anynan(probe_rotation):
continue
led_positions[i, :] = probe_tip[:]
break
athread.write_uint8(255, 0, 0, 0)
rig = fh.LedRig(rig_calibration_indices)
rotation, rig_led_positions = rig.solve(led_positions)
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
fig, axes = plt.subplots(3, 1)
def create_helmet(self):
head_measurement_points = [
'head straight', 'nasion', 'inion', 'right eye'
]
for i, measurement_point in enumerate(head_measurement_points):
print('Press space to measure: ' + measurement_point)
# light up an led to signal which measurement is going on
signal_led = 255
signal_length = 1
while True:
self.athread.write_uint8(
signal_led, 255, 255,
255) # bright light to start and see something
fh.wait_for_keypress(pygame.K_SPACE)
current_sample = self.othread.current_sample.copy()
helmet_leds = current_sample[HELMET].reshape((4, 3))
if np.any(np.isnan(helmet_leds)):
print('Helmet LEDs not all visible. Try again.')
self.athread.write_uint8(signal_led, 255, 0,
0) # red light for failure
time.sleep(signal_length)
continue
if i == I_BARY:
helmet = fh.Rigidbody(helmet_leds)
self.athread.write_uint8(signal_led, 0, 255,
0) # green light for success
time.sleep(signal_length)
break
else:
_, probe_tip = fh.FourMarkerProbe().solve(
current_sample[PROBE].reshape((4, 3)))
if np.any(np.isnan(probe_tip)):
print('Probe not visible. Try again.')
self.athread.write_uint8(signal_led, 255, 0,
0) # red light for failure
time.sleep(signal_length)
continue
helmet.add_reference_points(helmet_leds, probe_tip)
# replace eye measurement
if i == I_EYE:
nasion_to_inion = fh.to_unit(
helmet.ref_points[I_INION, :] -
helmet.ref_points[I_NASION, :])
# estimate eye at 15 mm inwards from probe in nasion inion direction
estimated_eye_position = helmet.ref_points[
I_EYE, :] + 15 * nasion_to_inion
# replace measured value with estimation
helmet.ref_points[I_EYE, :] = estimated_eye_position
self.athread.write_uint8(signal_led, 0, 255,
0) # green light for success
time.sleep(signal_length)
break
self.helmet = helmet
print('Helmet creation done.')
self.athread.write_uint8(255, 0, 0, 0) # turn off leds
def run_trial(
self,
trial_frame: pd.DataFrame) -> (TrialResult, Optional[OrderedDict]):
self.othread.reset_data_buffer()
self.pthread.reset_data_buffer()
self.athread.reset_command_timestamps()
# set up variables for one trial
left_to_right = trial_frame['left_to_right']
shift = trial_frame['shift'] if left_to_right else -trial_frame['shift']
amplitude = trial_frame[
'amplitude'] if left_to_right else -trial_frame['amplitude']
fixation_led = trial_frame[
'fixation_led'] if left_to_right else 254 - trial_frame[
'fixation_led']
target_led = fixation_led + amplitude
shifted_target_led = target_led + shift
fixation_threshold = trial_frame['fixation_threshold']
fixation_head_velocity_threshold = trial_frame[
'fixation_head_velocity_threshold']
saccade_threshold = trial_frame['saccade_threshold']
landing_fixation_threshold = trial_frame['landing_fixation_threshold']
pupil_min_confidence = trial_frame['pupil_min_confidence']
before_fixation_color = trial_frame['before_fixation_color']
during_fixation_color = trial_frame['during_fixation_color']
before_response_target_color = trial_frame[
'before_response_target_color']
during_response_target_color = trial_frame[
'during_response_target_color']
fixation_duration = trial_frame['fixation_duration']
blanking_duration = trial_frame['blanking_duration']
maximum_target_reaching_duration = trial_frame[
'maximum_target_reaching_duration']
maximum_saccade_latency = trial_frame['maximum_saccade_latency']
after_landing_fixation_duration = trial_frame[
'after_landing_fixation_duration']
inter_trial_interval = trial_frame['inter_trial_interval']
t_started_fixating = None
i_started_fixating = None
t_target_appeared = None
i_target_appeared = None
t_saccade_started = None
i_saccade_started = None
t_saccade_landed = None
i_saccade_landed = None
t_blanking_ended = None
i_blanking_ended = None
i_led_shift_done = None
i_target_turned_off = None
response = None
# turn off all leds
self.athread.write_uint8(255, 0, 0, 0)
# do the inter trial interval here, too many exit points
time.sleep(inter_trial_interval)
# show the fixation led
self.athread.write_uint8(fixation_led, *before_fixation_color)
phase = Phase.BEFORE_FIXATION
t_trial_started = time.monotonic()
last_i = None
R_head_world = np.full((3, 3), np.nan)
# this loop runs during data collection in the trial
# if trial_successful is true when you break out of it, the trial's parameters and timings are saved
trial_successful = False
while True:
# do calibration if escape was pressed
escape_pressed, backspace_pressed, kp_enter_pressed = fh.was_key_pressed(
pygame.K_ESCAPE, pygame.K_BACKSPACE, pygame.K_KP_ENTER)
if escape_pressed:
return TrialResult.CALIBRATE, None
if kp_enter_pressed:
return TrialResult.EYE_CALIBRATE, None
if backspace_pressed:
self.athread.write_uint8(255, 128, 0, 0)
key = fh.wait_for_keypress(pygame.K_ESCAPE, pygame.K_SPACE)
if key == pygame.K_ESCAPE:
return TrialResult.FAILED, None
else:
self.athread.write_uint8(255, 0, 128, 0)
time.sleep(0.5)
self.athread.write_uint8(255, 0, 0, 0)
return TrialResult.QUIT_EXPERIMENT, None
# check that a new pupil sample is available
current_i = self.pthread.i_current_sample
if current_i == last_i:
time.sleep(0.0005)
continue
last_i = current_i
pdata = self.pthread.current_sample.copy()
gaze_normals = pdata[NORMALS]
confidence = pdata[CONFIDENCE]
odata = self.othread.current_sample.copy()
helmet_leds = odata[HELMET].reshape((4, 3))
last_R_head_world = R_head_world
R_head_world, helmet_ref_points = self.helmet.solve(helmet_leds)
T_eye_world = helmet_ref_points[I_EYE, :]
# if helmet rigidbody couldn't be solved or pupil data is bad
if fh.anynan(R_head_world) or fh.anynan(gaze_normals) or (
confidence < pupil_min_confidence
and phase != Phase.DURING_SACCADE):
if phase == Phase.BEFORE_FIXATION:
continue
elif phase == Phase.DURING_FIXATION:
if fh.anynan(R_head_world):
print('head was not visible during fixation')
if fh.anynan(gaze_normals):
print('nan values in gaze normals during fixation')
if confidence < pupil_min_confidence:
print('pupil confidence was too low during fixation')
phase = Phase.BEFORE_FIXATION
self.athread.write_uint8(fixation_led,
*before_fixation_color)
continue
else:
if fh.anynan(R_head_world):
print('head was not visible')
if fh.anynan(gaze_normals):
print('nan values in gaze normals')
if confidence < pupil_min_confidence:
print('pupil confidence was too low')
break
current_head_angular_velocity = 0 if np.allclose(
last_R_head_world, R_head_world
) else np.rad2deg(
np.arccos(
(np.trace(last_R_head_world @ R_head_world.T) - 1) / 2)
) * self.othread.server_config['optotrak']['collection_frequency']
def is_eye_within_led_threshold(led, threshold):
eye_to_led = fh.to_unit(self.rig_leds[led, :] - T_eye_world)
gaze_normals_world = R_head_world @ fh.normals_nonlinear_angular_transform(
self.R_eye_head @ gaze_normals, self.nonlinear_parameters)
eye_to_led_azim_elev = np.rad2deg(
np.abs(
fh.to_azim_elev(gaze_normals_world) -
fh.to_azim_elev(eye_to_led)))
# threshold is only horizontal right now because of increased vertical angle noise and spikes
is_within_threshold = eye_to_led_azim_elev[0] <= threshold
return is_within_threshold
if phase == Phase.BEFORE_FIXATION:
is_fixating = is_eye_within_led_threshold(
fixation_led, fixation_threshold)
is_holding_still = current_head_angular_velocity <= fixation_head_velocity_threshold
if is_fixating and is_holding_still:
self.athread.write_uint8(fixation_led,
*during_fixation_color)
t_started_fixating = time.monotonic()
i_started_fixating = current_i
phase = Phase.DURING_FIXATION
elif phase == Phase.DURING_FIXATION:
is_fixating = is_eye_within_led_threshold(
fixation_led, fixation_threshold)
is_holding_still = current_head_angular_velocity <= fixation_head_velocity_threshold
if not (is_fixating and is_holding_still):
self.athread.write_uint8(fixation_led,
*before_fixation_color)
phase = Phase.BEFORE_FIXATION
# if fixation is lost here, don't start a completely new trial, that would be wasteful because
# the target wasn't even shown
else:
if time.monotonic(
) - t_started_fixating >= fixation_duration:
i_target_appeared = current_i
t_target_appeared = time.monotonic()
self.athread.write_uint8(target_led,
*before_response_target_color)
phase = Phase.BEFORE_SACCADE
elif phase == Phase.BEFORE_SACCADE:
if time.monotonic(
) - t_target_appeared > maximum_saccade_latency:
# abort trial because saccade latency was too long
print('maximum saccade latency exceeded')
break
has_started_saccade = not is_eye_within_led_threshold(
fixation_led, saccade_threshold)
if has_started_saccade:
i_saccade_started = current_i
t_saccade_started = time.monotonic()
if blanking_duration == 0:
i_led_shift_done = self.athread.write_uint8(
shifted_target_led, *before_response_target_color)
else:
i_target_turned_off = self.athread.write_uint8(
255, 0, 0, 0)
phase = Phase.DURING_SACCADE
elif phase == Phase.DURING_SACCADE:
if time.monotonic() - t_saccade_started >= blanking_duration:
if t_blanking_ended is None:
t_blanking_ended = time.monotonic()
i_blanking_ended = current_i
i_led_shift_done = self.athread.write_uint8(
shifted_target_led, *before_response_target_color)
if time.monotonic(
) - t_blanking_ended > maximum_target_reaching_duration:
print('maximum target reaching duration was exceeded')
break
is_fixating_target = is_eye_within_led_threshold(
shifted_target_led, landing_fixation_threshold)
if is_fixating_target:
i_saccade_landed = current_i
t_saccade_landed = time.monotonic()
phase = Phase.AFTER_LANDING
elif phase == Phase.AFTER_LANDING:
if time.monotonic(
) - t_saccade_landed > after_landing_fixation_duration:
self.athread.write_uint8(shifted_target_led,
*during_response_target_color)
response_key = fh.wait_for_keypress(
pygame.K_LEFT, pygame.K_RIGHT)
response = 'left' if response_key == pygame.K_LEFT else 'right'
trial_successful = True
break
# sampling loop over
if trial_successful:
trial_data = OrderedDict([
# arrays need to be wrapped in a list so pandas doesn't try to make them long columns
('o_data', [self.othread.get_shortened_data()]),
('p_data', [self.pthread.get_shortened_data()]),
('helmet', self.helmet),
('nonlinear_parameters', [self.nonlinear_parameters]),
('R_eye_head', [self.R_eye_head]),
('t_trial_started', t_trial_started),
('i_started_fixating', i_started_fixating),
('t_started_fixating', t_started_fixating),
('i_target_appeared', i_target_appeared),
('t_target_appeared', t_target_appeared),
('t_saccade_started', t_saccade_started),
('i_saccade_started', i_saccade_started),
('t_saccade_landed', t_saccade_landed),
('i_saccade_landed', i_saccade_landed),
('i_blanking_ended', i_blanking_ended),
('t_blanking_ended', t_blanking_ended),
('t_led_shift_done',
self.athread.command_timestamps[i_led_shift_done]),
('t_target_turned_off',
self.athread.command_timestamps[i_target_turned_off]
if blanking_duration > 0 else None),
('response', response),
])
return TrialResult.COMPLETED, trial_data
else:
return TrialResult.FAILED, None
def calibrate(self):
calibration_point = 127
self.athread.write_uint8(calibration_point, 128, 0, 0)
print('Press space to reset eye calibration.')
fh.wait_for_keypress(pygame.K_SPACE)
self.pthread.reset_3d_eye_model()
self.athread.write_uint8(calibration_point, 0, 0, 128)
time.sleep(self.after_reset_wait)
while True:
self.athread.write_uint8(calibration_point, 128, 0, 0)
print('Calibration pending. Press space to start.')
fh.wait_for_keypress(pygame.K_SPACE)
self.athread.write_uint8(calibration_point, 255, 0, 0)
self.othread.reset_data_buffer()
self.pthread.reset_data_buffer()
print('\nCalibration starting.\n')
start_time = time.monotonic()
while time.monotonic() - start_time < self.calib_duration:
time.sleep(0.1)
self.athread.write_uint8(255, 0, 0, 0)
print('\nCalibration over. Optimizing parameters...\n')
odata = self.othread.get_shortened_data().copy()
pdata = self.pthread.get_shortened_data().copy()
gaze_normals = pdata[:, NORMALS]
f_interpolate = interp1d(odata[:, OTIME],
odata[:, HELMET],
axis=0,
bounds_error=False,
fill_value=np.nan)
odata_interpolated = f_interpolate(pdata[:, PTIME]).reshape(
(-1, 4, 3))
R_head_world, ref_points = self.helmet.solve(odata_interpolated)
T_head_world = ref_points[:, I_BARY, :]
confidence_enough = pdata[:, CONFIDENCE] > 0.3
rotations_valid = ~np.any(np.isnan(R_head_world).reshape((-1, 9)),
axis=1)
chosen_mask = confidence_enough & rotations_valid
T_target_world = np.tile(self.rig_leds[calibration_point, :],
(chosen_mask.sum(), 1))
ini_T_eye_head = self.helmet.ref_points[
I_EYE, :] - self.helmet.ref_points[I_BARY, :]
calibration_result = fh.calibrate_pupil_nonlinear(
T_head_world[chosen_mask, ...],
R_head_world[chosen_mask, ...],
gaze_normals[chosen_mask, ...],
T_target_world,
ini_T_eye_head=ini_T_eye_head,
leave_T_eye_head=True)
print('Optimization done.\n')
print('Error: ', calibration_result.fun, '\n')
print('Parameters: ', calibration_result.x, '\n')
# signal quality of calibration via leds
if calibration_result.fun <= 0.7:
self.athread.write_uint8(127, 0, 255, 0) # green
elif 0.7 < calibration_result.fun <= 1:
self.athread.write_uint8(127, 255, 128, 0) # yellow
else:
self.athread.write_uint8(127, 255, 0, 0) # red
print('Accept calibration? Yes: Space, No: Escape')
key = fh.wait_for_keypress(pygame.K_SPACE, pygame.K_ESCAPE)
if key == pygame.K_SPACE:
self.R_eye_head = fh.from_yawpitchroll(
calibration_result.x[0:3])
self.nonlinear_parameters = calibration_result.x[3:9]
# self.helmet.ref_points[5, :] = self.helmet.ref_points[0, :] + calibration_result.x[9:12]
break
elif key == pygame.K_ESCAPE:
continue
self.athread.write_uint8(255, 0, 0, 0) # leds off
def extract_helmet(ot_data):
return ot_data[..., 3:15].reshape((-1, 4, 3)).squeeze()
def extract_gaze(p_data):
return p_data[3:6]
if not 'rig_led_positions' in locals():
led_rig_indices = [0, 70, 141, 213, 254] # these are the array indices
probe_tips = np.full((len(led_rig_indices), 3), np.nan)
for i, led_rig_index in enumerate(led_rig_indices):
athread.write_uint8(led_rig_index, 100, 0, 0)
print('Press space to record led with index', led_rig_index)
while True:
fh.wait_for_keypress(pygame.K_SPACE)
current_sample = othread.current_sample.copy()
rotation, tip = probe.solve(extract_probe(current_sample))
if np.any(np.isnan(tip)):
print('Probe not visible, try again.')
continue
probe_tips[i, :] = tip
break
athread.write_uint8(255, 0, 0, 0)
led_rig_transform_result = fh.get_rig_transform(probe_tips,
led_rig_indices)
R_rig = fh.from_yawpitchroll(led_rig_transform_result.x[0:3])
T_rig = led_rig_transform_result.x[3:6]
