def _draw_surface_frames(self, surface):
if not surface.detected:
return
(
corners,
top_indicator,
title_anchor,
surface_edit_anchor,
marker_edit_anchor,
) = self._get_surface_anchor_points(surface)
alpha = min(1, surface.build_up_status)
surface_color = rgb_to_rgba(self.color_primary_rgb, alpha=alpha)
pyglui_utils.draw_polyline(
corners.reshape((5, 2)), color=pyglui_utils.RGBA(*surface_color)
)
pyglui_utils.draw_polyline(
top_indicator.reshape((4, 2)), color=pyglui_utils.RGBA(*surface_color)
)
self._draw_surf_menu(
surface, title_anchor, surface_edit_anchor, marker_edit_anchor
)
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active or self.visualize:
# Draw hand detection results
for (x1, y1, x2, y2), fingertips in zip(self.hand_viz, self.finger_viz):
pts = np.array([[x1, y1], [x1, y2], [x2, y2], [x2, y1], [x1, y1]], np.int32)
cygl_utils.draw_polyline(pts, thickness=3 * self.g_pool.gui_user_scale, color=cygl_utils.RGBA(0., 1., 0., 1.))
for tip in fingertips:
if tip is not None:
y, x = tip
cygl_utils.draw_progress((x, y), 0., 1.,
inner_radius=25 * self.g_pool.gui_user_scale,
outer_radius=35 * self.g_pool.gui_user_scale,
color=cygl_utils.RGBA(1., 1., 1., 1.),
sharpness=0.9)
cygl_utils.draw_points([(x, y)], size=10 * self.g_pool.gui_user_scale,
color=cygl_utils.RGBA(1., 1., 1., 1.),
sharpness=0.9)
def _draw_surface_menu_buttons(self, surface, surface_edit_anchor,
marker_edit_anchor):
# Buttons
edit_button_color_rgba = rgb_to_rgba(self.color_primary_rgb)
edit_anchor_color_rgba = rgb_to_rgba(self.color_secondary_rgb)
text_color_rgba = rgb_to_rgba(self.color_secondary_rgb)
pyglui_utils.draw_points(
[marker_edit_anchor],
color=pyglui_utils.RGBA(*edit_button_color_rgba))
if surface in self._edit_surf_markers:
pyglui_utils.draw_points(
[marker_edit_anchor],
size=13,
color=pyglui_utils.RGBA(*edit_anchor_color_rgba),
)
pyglui_utils.draw_points(
[surface_edit_anchor],
color=pyglui_utils.RGBA(*edit_button_color_rgba))
if surface in self._edit_surf_corners:
pyglui_utils.draw_points(
[surface_edit_anchor],
size=13,
color=pyglui_utils.RGBA(*edit_anchor_color_rgba),
)
# Text
self._draw_text(
(surface_edit_anchor[0] + 15, surface_edit_anchor[1] + 6),
"edit surface",
text_color_rgba,
)
self._draw_text(
(marker_edit_anchor[0] + 15, marker_edit_anchor[1] + 6),
"add/remove markers",
text_color_rgba,
)
def _draw_marker_toggles(self, surface):
active_markers_by_type = {}
inactive_markers_by_type = {}
for marker in self.tracker.markers:
marker_type = marker.marker_type
if (
marker_type == Surface_Marker_Type.SQUARE
and marker.perimeter < self.tracker.marker_detector.marker_min_perimeter
):
continue
centroid = marker.centroid()
if marker.uid in surface.registered_markers_dist.keys():
active_markers = active_markers_by_type.get(marker_type, [])
active_markers.append(centroid)
active_markers_by_type[marker_type] = active_markers
else:
inactive_markers = inactive_markers_by_type.get(marker_type, [])
inactive_markers.append(centroid)
inactive_markers_by_type[marker_type] = inactive_markers
for marker_type, inactive_markers in inactive_markers_by_type.items():
color_rgb = SURFACE_MARKER_TOGGLE_INACTIVE_COLOR_RGB_BY_TYPE[marker_type]
color_rgba = rgb_to_rgba(color_rgb, alpha=0.8)
pyglui_utils.draw_points(
inactive_markers, size=20, color=pyglui_utils.RGBA(*color_rgba)
)
for marker_type, active_markers in active_markers_by_type.items():
color_rgb = SURFACE_MARKER_TOGGLE_ACTIVE_COLOR_RGB_BY_TYPE[marker_type]
color_rgba = rgb_to_rgba(color_rgb, alpha=0.8)
pyglui_utils.draw_points(
active_markers, size=20, color=pyglui_utils.RGBA(*color_rgba)
)
def draw_sections(self, width, height, scale):
t0, t1 = self.g_pool.timestamps[0], self.g_pool.timestamps[-1]
pixel_to_time_fac = height / (t1 - t0)
with gl_utils.Coord_System(t0, t1, height, 0):
gl.glTranslatef(0, 0.001 + scale * self.timeline_line_height / 2, 0)
for s in self.sections:
cal_slc = slice(*s['calibration_range'])
map_slc = slice(*s['mapping_range'])
cal_ts = self.g_pool.timestamps[cal_slc]
map_ts = self.g_pool.timestamps[map_slc]
color = cygl_utils.RGBA(*s['color'][:3], 1.)
if len(cal_ts):
cygl_utils.draw_rounded_rect((cal_ts[0], -4 * scale),
(cal_ts[-1] - cal_ts[0], 8 * scale),
corner_radius=0,
color=color,
sharpness=1.)
if len(map_ts):
cygl_utils.draw_rounded_rect((map_ts[0], -scale),
(map_ts[-1] - map_ts[0], 2 * scale),
corner_radius=0,
color=color,
sharpness=1.)
color = cygl_utils.RGBA(1., 1., 1., .5)
if s['calibration_method'] == "natural_features":
cygl_utils.draw_x([(m['timestamp'], 0) for m in self.manual_ref_positions],
height=12 * scale, width=3 * pixel_to_time_fac / scale, thickness=scale, color=color)
else:
cygl_utils.draw_bars([(m['timestamp'], 0) for m in self.circle_marker_positions],
height=12 * scale, thickness=scale, color=color)
gl.glTranslatef(0, scale * self.timeline_line_height, 0)
def _draw_bars_element_ts(self, element, scale, height):
color = cygl_utils.RGBA(*element.color_rgba)
cygl_utils.draw_bars(
[(ts, 0) for ts in element.bar_positions_ts],
height=element.height * scale,
thickness=element.width * scale,
color=color,
)
def _draw_current_reference(self, current_reference):
with self._frame_coordinate_system:
cygl_utils.draw_points(
[current_reference.screen_pos],
size=35,
color=cygl_utils.RGBA(0, 0.5, 0.5, 0.7),
)
self._draw_inner_dot(current_reference)
def _draw_markers(self):
for marker in self.tracker.markers_unfiltered:
color = rgb_to_rgba(
SURFACE_MARKER_COLOR_RGB_BY_TYPE[marker.marker_type],
alpha=0.5)
hat = np.array(
[[[0, 0], [0, 1], [0.5, 1.3], [1, 1], [1, 0], [0, 0]]],
dtype=np.float32)
hat = cv2.perspectiveTransform(
hat, _get_norm_to_points_trans(marker.verts_px))
# TODO: Should the had be drawn for small or low confidence markers?
pyglui_utils.draw_polyline(hat.reshape((6, 2)),
color=pyglui_utils.RGBA(*color))
if (marker.perimeter >= self.tracker.marker_min_perimeter and
marker.id_confidence > self.tracker.marker_min_confidence):
pyglui_utils.draw_polyline(hat.reshape((6, 2)),
color=pyglui_utils.RGBA(*color),
line_type=gl.GL_POLYGON)
def _draw_range(self, from_, to, scale, color_rgba, height, offset):
gl.glTranslatef(0, offset * scale, 0)
color = cygl_utils.RGBA(*color_rgba)
cygl_utils.draw_rounded_rect(
(from_, -height / 2 * scale),
(to - from_, height * scale),
corner_radius=0,
color=color,
sharpness=1.0,
)
gl.glTranslatef(0, -offset * scale, 0)
def _draw_surface_corner_handles(self, surface):
img_corners = surface.map_from_surf(
self.norm_corners.copy(),
self.tracker.camera_model,
compensate_distortion=False,
)
handle_color_rgba = rgb_to_rgba(self.color_primary_rgb, alpha=0.5)
pyglui_utils.draw_points(
img_corners, size=20, color=pyglui_utils.RGBA(*handle_color_rgba)
)
def _timeline_draw_data_cb(self, width, height, scale):
ts = self.g_pool.timestamps
with gl_utils.Coord_System(ts[0], ts[-1], height, 0):
# Lines for areas that have been cached
cached_ranges = []
for r in self.marker_cache.visited_ranges:
cached_ranges += ((ts[r[0]], 0), (ts[r[1]], 0))
gl.glTranslatef(0, scale * self.TIMELINE_LINE_HEIGHT / 2, 0)
color = pyglui_utils.RGBA(0.8, 0.2, 0.2, 0.8)
pyglui_utils.draw_polyline(cached_ranges,
color=color,
line_type=gl.GL_LINES,
thickness=scale * 4)
cached_ranges = []
for r in self.marker_cache.positive_ranges:
cached_ranges += ((ts[r[0]], 0), (ts[r[1]], 0))
color = pyglui_utils.RGBA(0, 0.7, 0.3, 0.8)
pyglui_utils.draw_polyline(cached_ranges,
color=color,
line_type=gl.GL_LINES,
thickness=scale * 4)
# Lines where surfaces have been found in video
cached_surfaces = []
for surface in self.surfaces:
found_at = []
if surface.location_cache is not None:
for r in surface.location_cache.positive_ranges: # [[0,1],[3,4]]
found_at += ((ts[r[0]], 0), (ts[r[1]], 0))
cached_surfaces.append(found_at)
color = pyglui_utils.RGBA(0, 0.7, 0.3, 0.8)
for surface in cached_surfaces:
gl.glTranslatef(0, scale * self.TIMELINE_LINE_HEIGHT, 0)
pyglui_utils.draw_polyline(surface,
color=color,
line_type=gl.GL_LINES,
thickness=scale * 2)
def draw_circle(self, segment: model.Classified_Segment):
segment_point = segment.last_2d_point_within_world(self._canvas_size)
circle_color = color_from_segment(segment).to_rgba().channels
gl_utils.draw_circle(
segment_point,
radius=48.0,
stroke_width=10.0,
color=gl_utils.RGBA(*circle_color),
)
self.draw_id(segment=segment, ref_point=segment_point)
def gl_display(self):
# normalize coordinate system, no need this step in utility functions
with gl_utils.Coord_System(0, 1, 0, 1):
ref_point_norm = [r['norm_pos'] for r in self.circle_marker_positions
if self.g_pool.capture.get_frame_index() == r['index']]
cygl_utils.draw_points(ref_point_norm, size=35, color=cygl_utils.RGBA(0, .5, 0.5, .7))
cygl_utils.draw_points(ref_point_norm, size=5, color=cygl_utils.RGBA(.0, .9, 0.0, 1.0))
manual_refs_in_frame = [r for r in self.manual_ref_positions
if self.g_pool.capture.get_frame_index() in r['index_range']]
current = self.g_pool.capture.get_frame_index()
for mr in manual_refs_in_frame:
if mr['index'] == current:
cygl_utils.draw_points([mr['norm_pos']], size=35, color=cygl_utils.RGBA(.0, .0, 0.9, .8))
cygl_utils.draw_points([mr['norm_pos']], size=5, color=cygl_utils.RGBA(.0, .9, 0.0, 1.0))
else:
distance = abs(current - mr['index'])
range_radius = (mr['index_range'][-1] - mr['index_range'][0]) // 2
# scale alpha [.1, .9] depending on distance to current frame
alpha = distance / range_radius
alpha = 0.1 * alpha + 0.9 * (1. - alpha)
# Use draw_progress instead of draw_circle. draw_circle breaks
# because of the normalized coord-system.
cygl_utils.draw_progress(mr['norm_pos'], 0., 0.999,
inner_radius=20.,
outer_radius=35.,
color=cygl_utils.RGBA(.0, .0, 0.9, alpha))
cygl_utils.draw_points([mr['norm_pos']], size=5, color=cygl_utils.RGBA(.0, .9, 0.0, alpha))
# calculate correct timeline height. Triggers timeline redraw only if changed
self.timeline.content_height = max(0.001, self.timeline_line_height * len(self.sections))
def _draw_surface_menu_buttons(
self, surface, surface_edit_anchor, marker_edit_anchor
):
# Buttons
edit_button_color_rgba = rgb_to_rgba(self.color_primary_rgb)
edit_anchor_color_rgba = rgb_to_rgba(self.color_secondary_rgb)
text_color_rgba = rgb_to_rgba(self.color_secondary_rgb)
self._draw_circle_filled(
tuple(marker_edit_anchor),
size=20 / 2,
color=pyglui_utils.RGBA(*edit_button_color_rgba),
)
if surface in self._edit_surf_markers:
self._draw_circle_filled(
tuple(marker_edit_anchor),
size=13 / 2,
color=pyglui_utils.RGBA(*edit_anchor_color_rgba),
)
self._draw_circle_filled(
tuple(surface_edit_anchor),
size=20 / 2,
color=pyglui_utils.RGBA(*edit_button_color_rgba),
)
if surface in self._edit_surf_corners:
self._draw_circle_filled(
tuple(surface_edit_anchor),
size=13 / 2,
color=pyglui_utils.RGBA(*edit_anchor_color_rgba),
)
# Text
self._draw_text(
(surface_edit_anchor[0] + 15, surface_edit_anchor[1] + 6),
"edit surface",
text_color_rgba,
)
self._draw_text(
(marker_edit_anchor[0] + 15, marker_edit_anchor[1] + 6),
"add/remove markers",
text_color_rgba,
)
def _draw_close_reference(self, reference_location, diff_to_current):
with self._frame_coordinate_system:
alpha = 0.7 * (1.0 - diff_to_current /
(self.close_ref_range + 1.0))
cygl_utils.draw_progress(
reference_location.screen_pos,
0.0,
0.999,
inner_radius=20.0,
outer_radius=35.0,
color=cygl_utils.RGBA(0, 0.5, 0.5, alpha),
)
self._draw_inner_dot(reference_location)
def _draw_marker_toggles(self, surface):
active_markers = []
inactive_markers = []
for marker in self.tracker.markers:
if marker.perimeter < self.tracker.marker_min_perimeter:
continue
centroid = np.mean(marker.verts_px, axis=0)
centroid = (centroid[0, 0], centroid[0, 1])
if marker.id in surface.registered_markers_dist.keys():
active_markers.append(centroid)
else:
inactive_markers.append(centroid)
pyglui_utils.draw_points(inactive_markers,
size=20,
color=pyglui_utils.RGBA(
*self.color_primary, 0.8))
pyglui_utils.draw_points(active_markers,
size=20,
color=pyglui_utils.RGBA(
*self.color_tertiary, 0.8))
def draw_recent_pupil_positions(self):
try:
for gp in self.surface.gaze_history:
pyglui_utils.draw_points(
[gp["norm_pos"]],
color=pyglui_utils.RGBA(0.0, 0.8, 0.5, 0.8),
size=80,
)
except AttributeError:
# If gaze_history does not exist, we are in the Surface_Tracker_Offline.
# In this case gaze visualizations will be drawn directly onto the scene
# image and thus propagate to the surface crop automatically.
pass
def _draw_surface_corner_handles(self, surface):
img_corners = surface.map_from_surf(
self.norm_corners.copy(),
self.tracker.camera_model,
compensate_distortion=False,
)
handle_color_rgba = rgb_to_rgba(self.color_primary_rgb, alpha=0.5)
for pt in img_corners:
self._draw_circle_filled(
tuple(pt),
size=20 / 2,
color=pyglui_utils.RGBA(*handle_color_rgba),
)
def draw_polyline(self, segment: model.Classified_Segment):
segment_points = segment.world_2d_points(self._canvas_size)
polyline_color = color_from_segment(segment).to_rgba().channels
polyline_thickness = 2
if not segment_points:
return
gl_utils.draw_polyline(
verts=segment_points,
thickness=float(polyline_thickness),
color=gl_utils.RGBA(*polyline_color),
)
self.draw_id(segment=segment, ref_point=segment_points[-1])
def _draw_markers(self):
color = pyglui_utils.RGBA(*self.color_secondary, 0.5)
for marker in self.tracker.markers_unfiltered:
hat = np.array(
[[[0, 0], [0, 1], [0.5, 1.3], [1, 1], [1, 0], [0, 0]]],
dtype=np.float32)
hat = cv2.perspectiveTransform(
hat, _get_norm_to_points_trans(marker.verts_px))
pyglui_utils.draw_polyline(hat.reshape((6, 2)), color=color)
if (marker.perimeter >= self.tracker.marker_min_perimeter and
marker.id_confidence > self.tracker.marker_min_confidence):
pyglui_utils.draw_polyline(hat.reshape((6, 2)),
color=color,
line_type=gl.GL_POLYGON)
import pyglui.cygl.utils as cygl_utils
from pyglui import ui
from pyglui.pyfontstash import fontstash as fs
from scipy.signal import fftconvolve
import csv_utils
import data_changed
import file_methods as fm
import gl_utils
import player_methods as pm
from observable import Observable
from plugin import Plugin
logger = logging.getLogger(__name__)
activity_color = cygl_utils.RGBA(0.6602, 0.8594, 0.4609, 0.8)
blink_color = cygl_utils.RGBA(0.9961, 0.3789, 0.5313, 0.8)
threshold_color = cygl_utils.RGBA(0.9961, 0.8438, 0.3984, 0.8)
class Blink_Detection(Plugin):
"""
This plugin implements a blink detection algorithm, based on sudden drops in the
pupil detection confidence.
"""
order = 0.8
icon_chr = chr(0xE81A)
icon_font = "pupil_icons"
@classmethod
def append_section_menu(self, sec):
section_menu = ui.Growing_Menu("Section Settings")
section_menu.color = cygl_utils.RGBA(*sec["color"])
def make_calibrate_fn(sec):
def calibrate():
self.calibrate_section(sec)
return calibrate
def make_remove_fn(sec):
def remove():
del self.menu[self.sections.index(sec) - len(self.sections)]
del self.sections[self.sections.index(sec)]
self.correlate_and_publish()
return remove
def set_trim_fn(button, sec, key):
def trim(format_only=False):
if format_only:
left_idx, right_idx = sec[key]
else:
right_idx = self.g_pool.seek_control.trim_right
left_idx = self.g_pool.seek_control.trim_left
sec[key] = left_idx, right_idx
time_fmt = key.replace("_", " ").split(" ")[0].title() + ": "
min_ts = self.g_pool.timestamps[0]
for idx in (left_idx, right_idx):
ts = self.g_pool.timestamps[idx] - min_ts
minutes = ts // 60
seconds = ts - (minutes * 60.0)
time_fmt += " {:02.0f}:{:02.0f} -".format(
abs(minutes), seconds)
button.outer_label = time_fmt[:-2] # remove final ' -'
button.function = trim
section_menu.append(ui.Text_Input("label", sec, label="Label"))
section_menu.append(
ui.Selector(
"calibration_method",
sec,
label="Calibration Method",
labels=["Circle Marker", "Natural Features"],
selection=["circle_marker", "natural_features"],
))
section_menu.append(
ui.Selector("mapping_method",
sec,
label="Calibration Mode",
selection=["2d", "3d"]))
section_menu.append(
ui.Text_Input("status",
sec,
label="Calibration Status",
setter=lambda _: _))
section_menu.append(
ui.Info_Text(
'This section is calibrated using reference markers found in a user set range "Calibration". The calibration is used to map pupil to gaze positions within a user set range "Mapping". Drag trim marks in the timeline to set a range and apply it.'
))
calib_range_button = ui.Button("Set from trim marks", None)
set_trim_fn(calib_range_button, sec, "calibration_range")
calib_range_button.function(format_only=True) # set initial label
section_menu.append(calib_range_button)
mapping_range_button = ui.Button("Set from trim marks", None)
set_trim_fn(mapping_range_button, sec, "mapping_range")
mapping_range_button.function(format_only=True) # set initial label
section_menu.append(mapping_range_button)
section_menu.append(ui.Button("Recalibrate", make_calibrate_fn(sec)))
section_menu.append(ui.Button("Remove section", make_remove_fn(sec)))
# manual gaze correction menu
offset_menu = ui.Growing_Menu("Manual Correction")
offset_menu.append(
ui.Info_Text("The manual correction feature allows you to apply" +
" a fixed offset to your gaze data."))
offset_menu.append(
ui.Slider("x_offset", sec, min=-0.5, step=0.01, max=0.5))
offset_menu.append(
ui.Slider("y_offset", sec, min=-0.5, step=0.01, max=0.5))
offset_menu.collapsed = True
section_menu.append(offset_menu)
self.menu.append(section_menu)
Lesser General Public License (LGPL v3.0).
See COPYING and COPYING.LESSER for license details.
---------------------------------------------------------------------------~(*)
"""
import numpy as np
import OpenGL.GL as gl
import pyglui.cygl.utils as cygl_utils
from pyglui import ui
from pyglui.pyfontstash import fontstash as fs
import data_changed
import gl_utils
from observable import Observable
from plugin import System_Plugin_Base
COLOR_LEGEND_WORLD = cygl_utils.RGBA(0.66, 0.86, 0.461, 1.0)
COLOR_LEGEND_EYE_RIGHT = cygl_utils.RGBA(0.9844, 0.5938, 0.4023, 1.0)
COLOR_LEGEND_EYE_LEFT = cygl_utils.RGBA(0.668, 0.6133, 0.9453, 1.0)
NUMBER_SAMPLES_TIMELINE = 4000
class System_Timelines(Observable, System_Plugin_Base):
def __init__(self, g_pool, show_world_fps=True, show_eye_fps=True):
super().__init__(g_pool)
self.show_world_fps = show_world_fps
self.show_eye_fps = show_eye_fps
self.cache_fps_data()
self.pupil_positions_listener = data_changed.Listener(
"pupil_positions", g_pool.rec_dir, plugin=self)
self.pupil_positions_listener.add_observer(
"on_data_changed", self._on_pupil_positions_changed)
import gl_utils
from audio_utils import Audio_Viz_Transform, NoAudioLoadedError, load_audio
from plugin import System_Plugin_Base
from version_utils import parse_version
assert parse_version(av.__version__) >= parse_version("0.4.4")
logger = logging.getLogger(__name__)
logger.setLevel(logger.DEBUG)
# av.logging.set_level(av.logging.DEBUG)
# logging.getLogger('libav').setLevel(logging.DEBUG)
viz_color = pyglui_utils.RGBA(0.9844, 0.5938, 0.4023, 1.0)
class FileSeekError(Exception):
pass
class Audio_Playback(System_Plugin_Base):
"""Calibrate using a marker on your screen
We use a ring detector that moves across the screen to 9 sites
Points are collected at sites not between
"""
icon_chr = chr(0xE050)
icon_font = "pupil_icons"
import OpenGL.GL as gl
import zmq
from pyglui import ui
import pyglui.cygl.utils as cygl_utils
from pyglui.pyfontstash import fontstash as fs
import file_methods as fm
import gl_utils
import player_methods as pm
import pupil_detectors # trigger module compilation
import zmq_tools
from plugin import Producer_Plugin_Base
logger = logging.getLogger(__name__)
COLOR_LEGEND_EYE_RIGHT = cygl_utils.RGBA(0.9844, 0.5938, 0.4023, 1.)
COLOR_LEGEND_EYE_LEFT = cygl_utils.RGBA(0.668, 0.6133, 0.9453, 1.)
NUMBER_SAMPLES_TIMELINE = 4000
class Empty(object):
pass
class Pupil_Producer_Base(Producer_Plugin_Base):
uniqueness = 'by_base_class'
order = 0.01
icon_chr = chr(0xec12)
icon_font = 'pupil_icons'
def init_ui(self):
def _draw_hat(points, color):
cygl_utils.draw_polyline(points, 1, cygl_utils.RGBA(*color), gl.GL_POLYGON)
def _draw_inner_dot(reference_location):
cygl_utils.draw_points(
[reference_location.screen_pos],
size=5,
color=cygl_utils.RGBA(0.0, 0.9, 0.0, 1.0),
)
class IMUTimeline(Plugin):
"""
plot and export imu data
export: imu_timeline.csv
keys:
imu_timestamp: timestamp of the source image frame
world_index: associated_frame: closest world video frame
gyro_x: angular velocity about the x axis in degrees/s
gyro_y: angular velocity about the y axis in degrees/s
gyro_z: angular velocity about the z axis in degrees/s
accel_x: linear acceleration along the x axis in G (9.80665 m/s^2)
accel_y: linear acceleration along the y axis in G (9.80665 m/s^2)
accel_z: linear acceleration along the z axis in G (9.80665 m/s^2)
pitch: orientation expressed as Euler angles
roll: orientation expressed as Euler angles
See Pupil docs for relevant coordinate systems
"""
IMU_PATTERN_RAW = r"^extimu ps(\d+).raw"
CMAP = {
"gyro_x": cygl_utils.RGBA(0.12156, 0.46666, 0.70588, 1.0),
"gyro_y": cygl_utils.RGBA(1.0, 0.49803, 0.05490, 1.0),
"gyro_z": cygl_utils.RGBA(0.17254, 0.62745, 0.1725, 1.0),
"accel_x": cygl_utils.RGBA(0.83921, 0.15294, 0.15686, 1.0),
"accel_y": cygl_utils.RGBA(0.58039, 0.40392, 0.74117, 1.0),
"accel_z": cygl_utils.RGBA(0.54901, 0.33725, 0.29411, 1.0),
"pitch": cygl_utils.RGBA(0.12156, 0.46666, 0.70588, 1.0),
"roll": cygl_utils.RGBA(1.0, 0.49803, 0.05490, 1.0),
}
NUMBER_SAMPLES_TIMELINE = 4000
TIMELINE_LINE_HEIGHT = 16
icon_chr = chr(0xEC22)
icon_font = "pupil_icons"
DTYPE_ORIENT = np.dtype(
[
("pitch", "<f4"),
("roll", "<f4"),
]
)
CACHE_VERSION = 1
@classmethod
def parse_pretty_class_name(cls) -> str:
return "IMU Timeline"
@classmethod
def is_available_within_context(cls, g_pool) -> bool:
if g_pool.app != "player":
# Plugin not available if not running in Player
return False
recording = PupilRecording(rec_dir=g_pool.rec_dir)
meta_info = recording.meta_info
if (
meta_info.recording_software_name
!= RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE
):
# Plugin not available if recording is not from Pupil Invisible
return False
imu_recs = cls._imu_recordings(g_pool)
if not len(imu_recs):
# Plugin not available if recording doesn't have IMU files (due to hardware failure, for example)
logger.debug(f"{cls.__name__} unavailable because there are no IMU files")
return False
return True
@classmethod
def _imu_recordings(cls, g_pool) -> T.List[IMURecording]:
rec = PupilRecording(g_pool.rec_dir)
imu_files: T.List[pathlib.Path] = sorted(
rec.files().filter_patterns(cls.IMU_PATTERN_RAW)
)
return [IMURecording(imu_file) for imu_file in imu_files]
def __init__(
self,
g_pool,
gyro_error=50,
should_draw_raw=True,
should_draw_orientation=True,
):
super().__init__(g_pool)
imu_recs = self._imu_recordings(g_pool)
# gyro_error settings priority
# 1. Loaded from cache (if available)
# 2. Loaded from session settings (if available)
# 3. Defaults to 50
self.gyro_error = gyro_error
self.should_draw_raw = should_draw_raw
self.should_draw_orientation = should_draw_orientation
self.bg_task = None
self.gyro_timeline = None
self.accel_timeline = None
self.orient_timeline = None
self.glfont_raw = None
self.glfont_orient = None
self.data_raw = np.concatenate([rec.raw for rec in imu_recs])
self.data_ts = np.concatenate([rec.ts for rec in imu_recs])
self.data_len = len(self.data_raw)
self.data_orient = self.data_orient_empty_copy()
self.read_orientation_cache()
self.gyro_keys = ["gyro_x", "gyro_y", "gyro_z"]
self.accel_keys = ["accel_x", "accel_y", "accel_z"]
self.orient_keys = ["pitch", "roll"]
def get_init_dict(self):
return {
"gyro_error": self.gyro_error,
"should_draw_raw": self.should_draw_raw,
"should_draw_orientation": self.should_draw_orientation,
}
def init_ui(self):
self.add_menu()
self.menu.label = "IMU Timeline"
self.menu.append(ui.Info_Text("Visualize IMU data and export to .csv file"))
self.menu.append(
ui.Info_Text(
"This plugin visualizes accelerometer, gyroscope and "
" orientation data from Pupil Invisible recordings. Results are "
" exported in 'imu_timeline.csv' "
)
)
self.menu.append(
ui.Info_Text(
"Orientation is estimated using Madgwick's algorithm. "
" Madgwick implements a beta value which is related with the "
" error of the gyroscope. Increasing the beta leads to faster "
" corrections but with more sensitivity to lateral accelerations. "
" Read more about Madgwick's algorithm here: "
" https://www.x-io.co.uk/res/doc/madgwick_internal_report.pdf "
)
)
def set_gyro_error(new_value):
self.gyro_error = new_value
self.notify_all({"subject": "madgwick_fusion.should_fuse", "delay": 0.3})
self.menu.append(
ui.Switch(
"should_draw_raw",
self,
label="View raw timeline",
setter=self.on_draw_raw_toggled,
)
)
self.menu.append(
ui.Switch(
"should_draw_orientation",
self,
label="View orientation timeline",
setter=self.on_draw_orientation_toggled,
)
)
self.menu.append(
ui.Slider(
"gyro_error",
self,
min=1,
step=0.1,
max=100,
label="Madgwick's beta",
setter=set_gyro_error,
)
)
if self.should_draw_raw:
self.append_timeline_raw()
if self.should_draw_orientation:
self.append_timeline_orientation()
if self.data_orient.shape[0] == 0:
# Start fusion after setting up timelines
self._fuse()
def deinit_ui(self):
if self.should_draw_raw:
self.g_pool.user_timelines.remove(self.gyro_timeline)
self.g_pool.user_timelines.remove(self.accel_timeline)
del self.gyro_timeline
del self.accel_timeline
del self.glfont_raw
if self.should_draw_orientation:
self.g_pool.user_timelines.remove(self.orient_timeline)
del self.glfont_orient
self.cleanup()
self.remove_menu()
def cleanup(self):
if self.bg_task:
self.bg_task.cancel()
self.bg_task = None
def _fuse(self):
"""
Fuse imu data
"""
if self.bg_task:
self.bg_task.cancel()
generator_args = (
self.data_raw,
self.gyro_error,
)
self.data_orient = self.data_orient_empty_copy()
self._data_orient_fetched = np.empty_like(self.data_orient, shape=self.data_len)
if self.should_draw_orientation:
self.orient_timeline.refresh()
logger.info("Starting IMU fusion using Madgwick's algorithm")
self.bg_task = bh.IPC_Logging_Task_Proxy("Fusion", fuser, args=generator_args)
def data_orient_empty_copy(self):
return np.empty([0], dtype=self.DTYPE_ORIENT).view(np.recarray)
def recent_events(self, events):
if self.bg_task:
start_time = time.perf_counter()
did_timeout = False
for progress, task_data in self.bg_task.fetch():
self.status = progress
if task_data:
current_progress = task_data[1] / self.data_len
self.menu_icon.indicator_stop = current_progress
self._data_orient_fetched["pitch"][task_data[1]] = task_data[0][0]
self._data_orient_fetched["roll"][task_data[1]] = task_data[0][1]
if time.perf_counter() - start_time > 1 / 50:
did_timeout = True
break
if self.bg_task.completed and not did_timeout:
self.status = "{} imu data fused"
self.bg_task = None
self.menu_icon.indicator_stop = 0.0
# swap orientation data buffers
self.data_orient = self._data_orient_fetched
del self._data_orient_fetched
if self.should_draw_orientation:
# redraw new orientation data
self.orient_timeline.refresh()
self.write_orientation_cache()
logger.info("Madgwick's fusion completed")
def on_draw_raw_toggled(self, new_value):
self.should_draw_raw = new_value
if self.should_draw_raw:
self.append_timeline_raw()
else:
self.remove_timeline_raw()
def on_draw_orientation_toggled(self, new_value):
self.should_draw_orientation = new_value
if self.should_draw_orientation:
self.append_timeline_orientation()
else:
self.remove_timeline_orientation()
def append_timeline_raw(self):
self.gyro_timeline = ui.Timeline(
"gyro",
self.draw_raw_gyro,
self.draw_legend_gyro,
self.TIMELINE_LINE_HEIGHT * 3,
)
self.accel_timeline = ui.Timeline(
"accel",
self.draw_raw_accel,
self.draw_legend_accel,
self.TIMELINE_LINE_HEIGHT * 3,
)
self.g_pool.user_timelines.append(self.gyro_timeline)
self.g_pool.user_timelines.append(self.accel_timeline)
self.glfont_raw = glfont_generator()
def append_timeline_orientation(self):
self.orient_timeline = ui.Timeline(
"orientation",
self.draw_orient,
self.draw_legend_orient,
self.TIMELINE_LINE_HEIGHT * 2,
)
self.g_pool.user_timelines.append(self.orient_timeline)
self.glfont_orient = glfont_generator()
def remove_timeline_raw(self):
self.g_pool.user_timelines.remove(self.gyro_timeline)
self.g_pool.user_timelines.remove(self.accel_timeline)
del self.gyro_timeline
del self.accel_timeline
del self.glfont_raw
def remove_timeline_orientation(self):
self.g_pool.user_timelines.remove(self.orient_timeline)
del self.glfont_orient
def draw_raw_gyro(self, width, height, scale):
y_limits = get_limits(self.data_raw, self.gyro_keys)
self._draw_grouped(
self.data_raw, self.gyro_keys, y_limits, width, height, scale
)
def draw_raw_accel(self, width, height, scale):
y_limits = get_limits(self.data_raw, self.accel_keys)
self._draw_grouped(
self.data_raw, self.accel_keys, y_limits, width, height, scale
)
def draw_orient(self, width, height, scale):
y_limits = get_limits(self.data_orient, self.orient_keys)
self._draw_grouped(
self.data_orient, self.orient_keys, y_limits, width, height, scale
)
def _draw_grouped(self, data, keys, y_limits, width, height, scale):
ts_min = self.g_pool.timestamps[0]
ts_max = self.g_pool.timestamps[-1]
data_raw = data[keys]
sub_samples = np.linspace(
0,
self.data_len - 1,
min(self.NUMBER_SAMPLES_TIMELINE, self.data_len),
dtype=int,
)
with gl_utils.Coord_System(ts_min, ts_max, *y_limits):
for key in keys:
data_keyed = data_raw[key]
if data_keyed.shape[0] == 0:
continue
points = list(zip(self.data_ts[sub_samples], data_keyed[sub_samples]))
cygl_utils.draw_points(points, size=1.5 * scale, color=self.CMAP[key])
def draw_legend_gyro(self, width, height, scale):
self._draw_legend_grouped(self.gyro_keys, width, height, scale, self.glfont_raw)
def draw_legend_accel(self, width, height, scale):
self._draw_legend_grouped(
self.accel_keys, width, height, scale, self.glfont_raw
)
def draw_legend_orient(self, width, height, scale):
self._draw_legend_grouped(
self.orient_keys, width, height, scale, self.glfont_orient
)
def _draw_legend_grouped(self, labels, width, height, scale, glfont):
glfont.set_size(self.TIMELINE_LINE_HEIGHT * 0.8 * scale)
pad = width * 2 / 3
for label in labels:
color = self.CMAP[label]
glfont.draw_text(width, 0, label)
cygl_utils.draw_polyline(
[
(pad, self.TIMELINE_LINE_HEIGHT / 2),
(width / 4, self.TIMELINE_LINE_HEIGHT / 2),
],
color=color,
line_type=gl.GL_LINES,
thickness=4.0 * scale,
)
gl.glTranslatef(0, self.TIMELINE_LINE_HEIGHT * scale, 0)
def on_notify(self, notification):
if notification["subject"] == "madgwick_fusion.should_fuse":
self._fuse()
elif notification["subject"] == "should_export":
if not self.bg_task:
self.export_data(notification["ts_window"], notification["export_dir"])
else:
logger.warning("Running Madgwick's algorithm")
def export_data(self, export_window, export_dir):
for_export = merge_arrays(self.data_raw, self.data_orient)
imu_bisector = Imu_Bisector(for_export, self.data_ts)
imu_exporter = Imu_Exporter()
imu_exporter.csv_export_write(
imu_bisector=imu_bisector,
timestamps=self.g_pool.timestamps,
export_window=export_window,
export_dir=export_dir,
)
def write_orientation_cache(self):
rec_dir = pathlib.Path(self.g_pool.rec_dir)
offline_data = rec_dir / "offline_data"
if not offline_data.exists():
offline_data.mkdir()
path_cache = offline_data / "orientation.cache"
path_meta = offline_data / "orientation.meta"
np.save(path_cache, self.data_orient)
fm.save_object(
{"version": self.CACHE_VERSION, "gyro_error": self.gyro_error}, path_meta
)
def read_orientation_cache(self) -> bool:
rec_dir = pathlib.Path(self.g_pool.rec_dir)
offline_data = rec_dir / "offline_data"
path_cache = offline_data / "orientation.cache.npy"
path_meta = offline_data / "orientation.meta"
if not (path_cache.exists() and path_meta.exists()):
return False
meta = fm.load_object(path_meta)
if meta["version"] != self.CACHE_VERSION:
return False
self.gyro_error = meta["gyro_error"]
self.data_orient = np.load(path_cache).view(np.recarray)
return True
def append_section_menu(self, sec):
section_menu = ui.Growing_Menu('Section Settings')
section_menu.color = cygl_utils.RGBA(*sec['color'])
def make_calibrate_fn(sec):
def calibrate():
self.calibrate_section(sec)
return calibrate
def make_remove_fn(sec):
def remove():
del self.menu[self.sections.index(sec) - len(self.sections)]
del self.sections[self.sections.index(sec)]
self.correlate_and_publish()
return remove
def set_trim_fn(button, sec, key):
def trim(format_only=False):
if format_only:
left_idx, right_idx = sec[key]
else:
right_idx = self.g_pool.seek_control.trim_right
left_idx = self.g_pool.seek_control.trim_left
sec[key] = left_idx, right_idx
time_fmt = key.replace('_', ' ').split(' ')[0].title() + ': '
min_ts = self.g_pool.timestamps[0]
for idx in (left_idx, right_idx):
ts = self.g_pool.timestamps[idx] - min_ts
minutes = ts // 60
seconds = ts - (minutes * 60.)
time_fmt += ' {:02.0f}:{:02.0f} -'.format(abs(minutes), seconds)
button.outer_label = time_fmt[:-2] # remove final ' -'
button.function = trim
section_menu.append(ui.Text_Input('label', sec, label='Label'))
section_menu.append(ui.Selector('calibration_method', sec,
label="Calibration Method",
labels=['Circle Marker', 'Natural Features'],
selection=['circle_marker', 'natural_features']))
section_menu.append(ui.Selector('mapping_method', sec, label='Calibration Mode',selection=['2d', '3d']))
section_menu.append(ui.Text_Input('status', sec, label='Calibration Status', setter=lambda _: _))
section_menu.append(ui.Info_Text('This section is calibrated using reference markers found in a user set range "Calibration". The calibration is used to map pupil to gaze positions within a user set range "Mapping". Drag trim marks in the timeline to set a range and apply it.'))
calib_range_button = ui.Button('Set from trim marks', None)
set_trim_fn(calib_range_button, sec, 'calibration_range')
calib_range_button.function(format_only=True) # set initial label
section_menu.append(calib_range_button)
mapping_range_button = ui.Button('Set from trim marks', None)
set_trim_fn(mapping_range_button, sec, 'mapping_range')
mapping_range_button.function(format_only=True) # set initial label
section_menu.append(mapping_range_button)
section_menu.append(ui.Button('Recalibrate', make_calibrate_fn(sec)))
section_menu.append(ui.Button('Remove section', make_remove_fn(sec)))
# manual gaze correction menu
offset_menu = ui.Growing_Menu('Manual Correction')
offset_menu.append(ui.Info_Text('The manual correction feature allows you to apply' +
' a fixed offset to your gaze data.'))
offset_menu.append(ui.Slider('x_offset', sec, min=-.5, step=0.01, max=.5))
offset_menu.append(ui.Slider('y_offset', sec, min=-.5, step=0.01, max=.5))
offset_menu.collapsed = True
section_menu.append(offset_menu)
self.menu.append(section_menu)