def __init__(
self,
g_pool,
marker_mode=None,
marker_scale=1.0,
sample_duration=40,
monitor_name=None,
**kwargs,
):
if marker_mode is None:
marker_mode = SingleMarkerMode.MANUAL
else:
marker_mode = SingleMarkerMode.from_label(marker_mode)
super().__init__(g_pool, **kwargs)
# Public properties
self.selected_monitor_name = monitor_name
self.marker_mode = marker_mode
# Private properties
self.__previously_detected_markers = []
self.__circle_tracker = CircleTracker()
self.__auto_stop_tracker = AutoStopTracker(
markers_needed=self._STOP_MARKER_FRAMES_NEEDED_TO_STOP
)
self.__marker_window = MarkerWindowController(marker_scale=marker_scale)
self.__marker_window.add_observer(
"on_window_did_close", self._on_window_did_close
)
def __init__(
self,
g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40,
monitor_name=None,
**kwargs,
):
super().__init__(g_pool, **kwargs)
# Public properties
self.selected_monitor_name = monitor_name
self.is_fullscreen = fullscreen
self.sample_duration = sample_duration
# Private properties
self.__current_list_of_markers_to_show = []
self.__currently_shown_marker_position = None
self.__ref_count_for_current_marker_position = 0
self.__previously_detected_markers = []
self.__circle_tracker = CircleTracker()
self.__marker_window = MarkerWindowController(marker_scale=marker_scale)
self.__marker_window.add_observer(
"on_window_did_close", self._on_window_did_close
)
def __init__(self, g_pool, fullscreen=True, marker_scale=1.0,
sample_duration=500, monitor_idx=0):
super().__init__(g_pool)
self.screen_marker_state = 0.
self.sample_duration = sample_duration # number of frames to sample per site
self.lead_in = 100 # frames of marker shown before starting to sample
self.lead_out = 15 # frames of markers shown after sampling is donw
self.active_site = None
self.sites = []
self.display_pos = -1., -1.
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.monitor_idx = monitor_idx
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center')
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def __init__(self, g_pool,fullscreen=True,marker_scale=1.0,sample_duration=40):
super().__init__(g_pool)
self.screen_marker_state = 0.
self.lead_in = 25 # frames of marker shown before starting to sample
self.display_pos = (.5,.5)
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center')
# UI Platform tweaks
if system() == 'Linux':
self.window_position_default = (0, 0)
elif system() == 'Windows':
self.window_position_default = (8, 31)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def __init__(
self,
g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.sample_duration = sample_duration # number of frames to sample per site
self.lead_in = 25 # frames of marker shown before starting to sample
self.lead_out = 5 # frames of markers shown after sampling is donw
self.active_site = None
self.sites = []
self.display_pos = -1.0, -1.0
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.monitor_idx = monitor_idx
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def __init__(
self,
g_pool,
marker_mode="Full screen",
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.lead_in = 25 # frames of marker shown before starting to sample
self.display_pos = (0.5, 0.5)
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.monitor_idx = monitor_idx
self.marker_mode = marker_mode
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def __init__(self, g_pool):
super().__init__(g_pool)
self.pos = None
self.smooth_pos = 0., 0.
self.smooth_vel = 0.
self.sample_site = (-2, -2)
self.counter = 0
self.counter_max = 30
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.menu = None
self.circle_tracker = CircleTracker()
self.markers = []
def __init__(
self,
g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.sample_duration = sample_duration # number of frames to sample per site
self.lead_in = 25 # frames of marker shown before starting to sample
self.lead_out = 5 # frames of markers shown after sampling is donw
self.active_site = None
self.sites = []
self.display_pos = -1.0, -1.0
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.monitor_idx = monitor_idx
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def __init__(
self,
g_pool,
marker_mode="Full screen",
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.lead_in = 25 # frames of marker shown before starting to sample
self.display_pos = (0.5, 0.5)
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.monitor_idx = monitor_idx
self.marker_mode = marker_mode
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def __init__(self, g_pool):
super().__init__(g_pool)
self.pos = None
self.smooth_pos = 0.0, 0.0
self.smooth_vel = 0.0
self.sample_site = (-2, -2)
self.counter = 0
self.counter_max = 30
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.menu = None
self.circle_tracker = CircleTracker()
self.markers = []
class Manual_Marker_Calibration(Calibration_Plugin):
"""
CircleTracker looks for proper markers
Using at least 9 positions/points within the FOV
Ref detector will direct one to good positions with audio cues
Calibration only collects data at the good positions
"""
def __init__(self, g_pool):
super().__init__(g_pool)
self.pos = None
self.smooth_pos = 0.0, 0.0
self.smooth_vel = 0.0
self.sample_site = (-2, -2)
self.counter = 0
self.counter_max = 30
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.menu = None
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.menu.label = "Manual Calibration"
self.menu.append(
ui.Info_Text("Calibrate gaze parameters using a handheld marker.")
)
def start(self):
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
self.active = True
self.ref_list = []
self.pupil_list = []
def stop(self):
audio.say("Stopping {}".format(self.mode_pretty))
logger.info("Stopping {}".format(self.mode_pretty))
self.screen_marker_state = 0
self.active = False
self.smooth_pos = 0.0, 0.0
# self.close_window()
self.button.status_text = ""
if self.mode == "calibration":
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == "accuracy_test":
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def on_notify(self, notification):
"""
Reacts to notifications:
``calibration.should_start``: Starts the calibration procedure
``calibration.should_stop``: Stops the calibration procedure
Emits notifications:
``calibration.started``: Calibration procedure started
``calibration.stopped``: Calibration procedure stopped
``calibration.marker_found``: Steady marker found
``calibration.marker_moved_too_quickly``: Marker moved too quickly
``calibration.marker_sample_completed``: Enough data points sampled
"""
super().on_notify(notification)
def recent_events(self, events):
"""
gets called once every frame.
reference positon need to be published to shared_pos
if no reference was found, publish 0,0
"""
frame = events.get("frame")
if self.active and frame:
gray_img = frame.gray
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
self.stop_marker_found = False
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]["img_pos"]
self.pos = self.markers[0]["norm_pos"]
# Check if there are stop markers
for marker in self.markers:
if marker["marker_type"] == "Stop":
self.auto_stop += 1
self.stop_marker_found = True
break
else:
self.pos = None # indicate that no reference is detected
if self.stop_marker_found is False:
self.auto_stop = 0
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".format(
len(self.markers)
)
)
# tracking logic
if len(self.markers) and not self.stop_marker_found:
# start counter if ref is resting in place and not at last sample site
# calculate smoothed manhattan velocity
smoother = 0.3
smooth_pos = np.array(self.smooth_pos)
pos = np.array(self.pos)
new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
smooth_vel_vec = new_smooth_pos - smooth_pos
smooth_pos = new_smooth_pos
self.smooth_pos = list(smooth_pos)
# manhattan distance for velocity
new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
self.smooth_vel = self.smooth_vel + smoother * (
new_vel - self.smooth_vel
)
# distance to last sampled site
sample_ref_dist = smooth_pos - np.array(self.sample_site)
sample_ref_dist = abs(sample_ref_dist[0]) + abs(sample_ref_dist[1])
# start counter if ref is resting in place and not at last sample site
if self.counter <= 0:
if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
self.sample_site = self.smooth_pos
audio.beep()
logger.debug(
"Steady marker found. Starting to sample {} datapoints".format(
self.counter_max
)
)
self.notify_all(
{
"subject": "calibration.marker_found",
"timestamp": self.g_pool.get_timestamp(),
"record": True,
}
)
self.counter = self.counter_max
if self.counter > 0:
if self.smooth_vel > 0.01:
audio.tink()
logger.warning(
"Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again.".format(
self.counter_max - self.counter
)
)
self.notify_all(
{
"subject": "calibration.marker_moved_too_quickly",
"timestamp": self.g_pool.get_timestamp(),
"record": True,
}
)
self.counter = 0
else:
self.counter -= 1
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
if events.get("fixations", []):
self.counter -= 5
if self.counter <= 0:
# last sample before counter done and moving on
audio.tink()
logger.debug(
"Sampled {} datapoints. Stopping to sample. Looking for steady marker again.".format(
self.counter_max
)
)
self.notify_all(
{
"subject": "calibration.marker_sample_completed",
"timestamp": self.g_pool.get_timestamp(),
"record": True,
}
)
# Always save pupil positions
self.pupil_list.extend(events["pupil"])
if self.counter:
if len(self.markers):
self.button.status_text = "Sampling Gaze Data"
else:
self.button.status_text = "Marker Lost"
else:
self.button.status_text = "Looking for Marker"
# Stop if autostop condition is satisfied:
if self.auto_stop >= self.auto_stop_max:
self.auto_stop = 0
self.stop()
else:
pass
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active:
draw_points_norm([self.smooth_pos], size=15, color=RGBA(1.0, 1.0, 0.0, 0.5))
if self.active and len(self.markers):
# draw the largest ellipse of all detected markers
for marker in self.markers:
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0))
if len(self.markers) > 1:
draw_polyline(
pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=GL_POLYGON
)
# draw indicator on the first detected marker
if self.counter and self.markers[0]["marker_type"] == "Ref":
e = self.markers[0]["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self.counter_max,
)
indicator = [e[0]] + pts[self.counter :].tolist()[::-1] + [e[0]]
draw_polyline(
indicator, color=RGBA(0.1, 0.5, 0.7, 0.8), line_type=GL_POLYGON
)
# draw indicator on the stop marker(s)
if self.auto_stop:
for marker in self.markers:
if marker["marker_type"] == "Stop":
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self.auto_stop_max,
)
indicator = [e[0]] + pts[self.auto_stop :].tolist() + [e[0]]
draw_polyline(
indicator,
color=RGBA(8.0, 0.1, 0.1, 0.8),
line_type=GL_POLYGON,
)
else:
pass
def deinit_ui(self):
"""gets called when the plugin get terminated.
This happens either voluntarily or forced.
if you have an atb bar or glfw window destroy it here.
"""
if self.active:
self.stop()
super().deinit_ui()
import cv2
import numpy as np
import os
import timeit
from circle_detector import CircleTracker
photo_record_mode = True
Test_folder = "2017_11_30-002-world"
if photo_record_mode:
marked_image_folder = "{0}/{0}-marked".format(Test_folder)
if not os.path.exists(marked_image_folder):
os.makedirs(marked_image_folder)
circle_tracker = CircleTracker(wait_interval=1)
duration = []
image_count = 0
for root, dirs, files in os.walk(Test_folder):
if "marked" in root:
continue
files.sort()
for file in files:
if os.path.splitext(file)[-1] != '.jpg':
continue
if "2017_11_30-002-world-frame-00000" not in file:
continue
photo_name = os.path.splitext(file)[0]
photo_path = os.path.join(root, file)
class Single_Marker_Calibration(Calibration_Plugin):
"""Calibrate using a single marker.
Move your head for example in a spiral motion while gazing
at the marker to quickly sample a wide range gaze angles.
"""
def __init__(
self,
g_pool,
marker_mode="Full screen",
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.lead_in = 25 # frames of marker shown before starting to sample
self.display_pos = (0.5, 0.5)
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.monitor_idx = monitor_idx
self.marker_mode = marker_mode
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
def get_monitors_idx_list():
monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
return range(len(monitors)), monitors
if self.monitor_idx not in get_monitors_idx_list()[0]:
logger.warning(
"Monitor at index %s no longer availalbe using default" % idx)
self.monitor_idx = 0
self.menu.append(
ui.Info_Text(
"Calibrate using a single marker. Gaze at the center of the marker and move your head (e.g. in a slow spiral movement). This calibration method enables you to quickly sample a wide range of gaze angles and cover a large range of your FOV."
))
self.menu.append(
ui.Selector(
"marker_mode",
self,
selection=["Full screen", "Window", "Manual"],
label="Marker display mode",
))
self.menu.append(
ui.Selector(
"monitor_idx",
self,
selection_getter=get_monitors_idx_list,
label="Monitor",
))
self.menu.append(
ui.Slider("marker_scale",
self,
step=0.1,
min=0.5,
max=2.0,
label="Marker size"))
def start(self):
if not self.g_pool.capture.online:
logger.error(
"This calibration requires world capture video input.")
return
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
self.active = True
self.ref_list = []
self.pupil_list = []
self.clicks_to_close = 5
if self.marker_mode != "Manual":
self.open_window(self.mode_pretty)
def open_window(self, title="new_window"):
if not self._window:
if self.marker_mode == "Full screen":
try:
monitor = glfwGetMonitors()[self.monitor_idx]
except Exception:
logger.warning(
"Monitor at index %s no longer availalbe using default"
% idx)
self.monitor_idx = 0
monitor = glfwGetMonitors()[self.monitor_idx]
width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(
monitor)
else:
monitor = None
width, height = 640, 360
self._window = glfwCreateWindow(width,
height,
title,
monitor=monitor,
share=glfwGetCurrentContext())
if self.marker_mode == "Window":
glfwSetWindowPos(
self._window,
self.window_position_default[0],
self.window_position_default[1],
)
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
# Register callbacks
glfwSetFramebufferSizeCallback(self._window, on_resize)
glfwSetKeyCallback(self._window, self.on_window_key)
glfwSetMouseButtonCallback(self._window,
self.on_window_mouse_button)
on_resize(self._window, *glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
glfwMakeContextCurrent(active_window)
def on_window_key(self, window, key, scancode, action, mods):
if action == GLFW_PRESS:
if self.mode == "calibration":
target_key = GLFW_KEY_C
else:
target_key = GLFW_KEY_T
if key == GLFW_KEY_ESCAPE or key == target_key:
self.clicks_to_close = 0
def on_window_mouse_button(self, window, button, action, mods):
if action == GLFW_PRESS:
self.clicks_to_close -= 1
def stop(self):
# TODO: redundancy between all gaze mappers -> might be moved to parent class
audio.say("Stopping {}".format(self.mode_pretty))
logger.info("Stopping {}".format(self.mode_pretty))
self.smooth_pos = 0, 0
self.counter = 0
self.close_window()
self.active = False
self.button.status_text = ""
if self.mode == "calibration":
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == "accuracy_test":
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def close_window(self):
if self._window:
# enable mouse display
active_window = glfwGetCurrentContext()
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL)
glfwDestroyWindow(self._window)
self._window = None
glfwMakeContextCurrent(active_window)
def recent_events(self, events):
frame = events.get("frame")
if self.active and frame:
gray_img = frame.gray
if self.clicks_to_close <= 0:
self.stop()
return
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
self.stop_marker_found = False
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]["img_pos"]
self.pos = self.markers[0]["norm_pos"]
# Check if there are stop markers
for marker in self.markers:
if marker["marker_type"] == "Stop":
self.auto_stop += 1
self.stop_marker_found = True
break
else:
self.pos = None # indicate that no reference is detected
if self.stop_marker_found is False:
self.auto_stop = 0
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".
format(len(self.markers)))
# only save a valid ref position if within sample window of calibraiton routine
on_position = self.lead_in < self.screen_marker_state
if on_position and len(
self.markers) and not self.stop_marker_found:
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# always save pupil positions
self.pupil_list.extend(events["pupil"])
# Animate the screen marker
if len(self.markers) or not on_position:
self.screen_marker_state += 1
# Stop if autostop condition is satisfied:
if self.auto_stop >= self.auto_stop_max:
self.auto_stop = 0
self.stop()
# use np.arrays for per element wise math
self.on_position = on_position
if self._window:
self.gl_display_in_window()
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active:
# draw the largest ellipse of all detected markers
for marker in self.markers:
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0))
if len(self.markers) > 1:
draw_polyline(pts,
1,
RGBA(1.0, 0.0, 0.0, 0.5),
line_type=gl.GL_POLYGON)
# draw indicator on the stop marker(s)
if self.auto_stop:
for marker in self.markers:
if marker["marker_type"] == "Stop":
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self.auto_stop_max,
)
indicator = [e[0]] + pts[self.auto_stop:].tolist() + [
e[0]
]
draw_polyline(
indicator,
color=RGBA(8.0, 0.1, 0.1, 0.8),
line_type=gl.GL_POLYGON,
)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = getHDPIFactor(self._window)
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = (
map_value(self.display_pos[0],
out_range=(pad, p_window_size[0] - pad)),
map_value(self.display_pos[1],
out_range=(p_window_size[1] - pad, pad)),
)
alpha = (
1.0
) # interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
r2 = 2 * r
draw_points([screen_pos],
size=60 * r2,
color=RGBA(0.0, 0.0, 0.0, alpha),
sharpness=0.9)
draw_points([screen_pos],
size=38 * r2,
color=RGBA(1.0, 1.0, 1.0, alpha),
sharpness=0.8)
draw_points([screen_pos],
size=19 * r2,
color=RGBA(0.0, 0.0, 0.0, alpha),
sharpness=0.55)
# some feedback on the detection state
color = (RGBA(0.0, 0.8, 0.0, alpha) if len(self.markers)
and self.on_position else RGBA(0.8, 0.0, 0.0, alpha))
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch {} more times to cancel calibration.".format(
self.clicks_to_close),
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def get_init_dict(self):
d = {}
d["marker_mode"] = self.marker_mode
d["marker_scale"] = self.marker_scale
d["monitor_idx"] = self.monitor_idx
return d
def deinit_ui(self):
"""gets called when the plugin get terminated.
either voluntarily or forced.
"""
if self.active:
self.stop()
if self._window:
self.close_window()
super().deinit_ui()
class Screen_Marker_Calibration(Calibration_Plugin):
"""Calibrate using a marker on your screen
We use a ring detector that moves across the screen to 9 sites
Points are collected at sites - not between
"""
def __init__(self, g_pool, fullscreen=True, marker_scale=1.0,
sample_duration=500, monitor_idx=0):
super().__init__(g_pool)
self.screen_marker_state = 0.
self.sample_duration = sample_duration # number of frames to sample per site
self.lead_in = 100 # frames of marker shown before starting to sample
self.lead_out = 15 # frames of markers shown after sampling is donw
self.active_site = None
self.sites = []
self.display_pos = -1., -1.
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.monitor_idx = monitor_idx
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center')
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.menu.label = "Screen Marker Calibration with Jerk Nystagmus"
def get_monitors_idx_list():
monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
return range(len(monitors)),monitors
if self.monitor_idx not in get_monitors_idx_list()[0]:
logger.warning("Monitor at index %s no longer availalbe using default"%self.monitor_idx)
self.monitor_idx = 0
self.menu.append(ui.Info_Text("Calibrate gaze parameters using a screen based animation."))
self.menu.append(ui.Selector('monitor_idx',self,selection_getter = get_monitors_idx_list,label='Monitor'))
self.menu.append(ui.Switch('fullscreen',self,label='Use fullscreen'))
self.menu.append(ui.Slider('marker_scale',self,step=0.1,min=0.5,max=2.0,label='Marker size'))
self.menu.append(ui.Slider('sample_duration',self,step=1,min=10,max=100,label='Sample duration'))
def start(self):
if not self.g_pool.capture.online:
logger.error("{} requiers world capture video input.".format(self.mode_pretty))
return
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
if self.mode == 'calibration':
self.sites = [(.25, .5), (0, .5), (0., 1.), (.5, 1.), (1., 1.),
(1., .5), (1., 0.), (.5, 0.), (0., 0.), (.75, .5)]
else:
self.sites = [(.5, .5), (.25, .25), (.25, .75), (.75, .75), (.75, .25)]
self.active_site = self.sites.pop(0)
self.active = True
self.ref_list = []
self.pupil_list = []
self.clicks_to_close = 5
self.open_window(self.mode_pretty)
def open_window(self, title='new_window'):
if not self._window:
if self.fullscreen:
try:
monitor = glfwGetMonitors()[self.monitor_idx]
except:
logger.warning("Monitor at index %s no longer availalbe using default"%self.monitor_idx)
self.monitor_idx = 0
monitor = glfwGetMonitors()[self.monitor_idx]
width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(monitor)
else:
monitor = None
width,height= 640,360
self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext())
if not self.fullscreen:
glfwSetWindowPos(self._window, self.window_position_default[0], self.window_position_default[1])
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
# Register callbacks
glfwSetFramebufferSizeCallback(self._window, on_resize)
glfwSetKeyCallback(self._window, self.on_window_key)
glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)
on_resize(self._window, *glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
glfwMakeContextCurrent(active_window)
def on_window_key(self,window, key, scancode, action, mods):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
self.clicks_to_close = 0
def on_window_mouse_button(self,window,button, action, mods):
if action ==GLFW_PRESS:
self.clicks_to_close -=1
def stop(self):
# TODO: redundancy between all gaze mappers -> might be moved to parent class
audio.say("Stopping {}".format(self.mode_pretty))
logger.info("Stopping {}".format(self.mode_pretty))
self.smooth_pos = 0, 0
self.counter = 0
self.close_window()
self.active = False
self.button.status_text = ''
fixation_list = is_fixation(self)
if self.mode == 'calibration':
finish_calibration(self.g_pool, self.fixation_list, self.ref_list)
elif self.mode == 'accuracy_test':
self.finish_accuracy_test(self.fixation_list, self.ref_list)
super().stop()
def close_window(self):
if self._window:
# enable mouse display
active_window = glfwGetCurrentContext()
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL)
glfwDestroyWindow(self._window)
self._window = None
glfwMakeContextCurrent(active_window)
def recent_events(self, events):
frame = events.get('frame')
if self.active and frame:
gray_img = frame.gray
if self.clicks_to_close <=0:
self.stop()
return
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
# Screen marker takes only Ref marker
self.markers = [marker for marker in self.markers if marker['marker_type'] == 'Ref']
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]['img_pos']
self.pos = self.markers[0]['norm_pos']
else:
self.pos = None # indicate that no reference is detected
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning("{} markers detected. Please remove all the other markers".format(len(self.markers)))
# only save a valid ref position if within sample window of calibration routine
on_position = self.lead_in < self.screen_marker_state < (self.lead_in+self.sample_duration)
if on_position and len(self.markers):
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# Always save pupil positions
self.pupil_list.extend(events['pupil_positions'])
'''
if on_position and len(self.markers) and events.get('fixations', []):
fixation_boost = 5
self.screen_marker_state = min(
self.sample_duration+self.lead_in,
self.screen_marker_state+fixation_boost)
'''
# Animate the screen marker
if self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out:
if len(self.markers) or not on_position:
self.screen_marker_state += 1
else:
self.screen_marker_state = 0
if not self.sites:
self.stop()
return
self.active_site = self.sites.pop(0)
logger.debug("Moving screen marker to site at {} {}".format(*self.active_site))
# use np.arrays for per element wise math
self.display_pos = np.array(self.active_site)
self.on_position = on_position
self.button.status_text = '{}'.format(self.active_site)
if self._window:
self.gl_display_in_window()
class SingleMarkerChoreographyPlugin(
MonitorSelectionMixin, CalibrationChoreographyPlugin
):
"""Calibrate using a single marker.
Move your head for example in a spiral motion while gazing
at the marker to quickly sample a wide range gaze angles.
"""
label = "Single Marker Calibration"
@classmethod
def selection_label(cls) -> str:
return "Single Marker"
@classmethod
def selection_order(cls) -> float:
return 2.0
_STOP_MARKER_FRAMES_NEEDED_TO_STOP = 30
_FIXED_MARKER_POSITION = (0.5, 0.5)
def __init__(
self,
g_pool,
marker_mode=None,
marker_scale=1.0,
sample_duration=40,
monitor_name=None,
**kwargs,
):
if marker_mode is None:
marker_mode = SingleMarkerMode.MANUAL
else:
marker_mode = SingleMarkerMode.from_label(marker_mode)
super().__init__(g_pool, **kwargs)
# Public properties
self.selected_monitor_name = monitor_name
self.marker_mode = marker_mode
# Private properties
self.__previously_detected_markers = []
self.__circle_tracker = CircleTracker()
self.__auto_stop_tracker = AutoStopTracker(
markers_needed=self._STOP_MARKER_FRAMES_NEEDED_TO_STOP
)
self.__marker_window = MarkerWindowController(marker_scale=marker_scale)
self.__marker_window.add_observer(
"on_window_did_close", self._on_window_did_close
)
def get_init_dict(self):
d = {}
d["marker_mode"] = self.marker_mode.label
d["marker_scale"] = self.__marker_window.marker_scale
d["monitor_name"] = self.selected_monitor_name
return d
def cleanup(self):
super().cleanup()
@property
def marker_mode(self) -> SingleMarkerMode:
return self.__marker_mode
@marker_mode.setter
def marker_mode(self, value: SingleMarkerMode):
self.__marker_mode = value
self._ui_update_visibility_digital_marker_config()
### Public - Plugin
@classmethod
def _choreography_description_text(cls) -> str:
return "Calibrate using a single marker. Gaze at the center of the marker and move your head (e.g. in a slow spiral movement). This calibration method enables you to quickly sample a wide range of gaze angles and cover a large range of your FOV."
def _init_custom_menu_ui_elements(self) -> list:
self.__ui_selector_marker_mode = ui.Selector(
"marker_mode",
self,
label="Marker display mode",
labels=[m.label for m in SingleMarkerMode.all_modes()],
selection=SingleMarkerMode.all_modes(),
)
# TODO: potential race condition through selection_getter. Should ensure
# that current selection will always be present in the list returned by the
# selection_getter. Highly unlikely though as this needs to happen between
# having clicked the Selector and the next redraw.
# See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b
self.__ui_selector_monitor_name = ui.Selector(
"selected_monitor_name",
self,
label="Monitor",
labels=self.currently_connected_monitor_names(),
selection=self.currently_connected_monitor_names(),
)
self.__ui_slider_marker_scale = ui.Slider(
"marker_scale",
self.__marker_window,
label="Marker size",
min=0.5,
max=2.0,
step=0.1,
)
return [
self.__ui_selector_marker_mode,
self.__ui_selector_monitor_name,
self.__ui_slider_marker_scale,
]
def init_ui(self):
super().init_ui()
# Save UI elements that are part of the digital marker config
self.__ui_digital_marker_config_elements = [
self.__ui_selector_monitor_name,
self.__ui_slider_marker_scale,
]
# Save start index of the UI elements of digital marker config
self.__ui_digital_marker_config_start_index = min(
self.menu.elements.index(elem)
for elem in self.__ui_digital_marker_config_elements
)
self._ui_update_visibility_digital_marker_config()
def _ui_update_visibility_digital_marker_config(self):
try:
ui_menu = self.menu
ui_elements = self.__ui_digital_marker_config_elements
start_index = self.__ui_digital_marker_config_start_index
except AttributeError:
return
is_visible = self.marker_mode != SingleMarkerMode.MANUAL
for i, ui_element in enumerate(ui_elements):
index = start_index + i
if is_visible and ui_element not in ui_menu:
ui_menu.insert(index, ui_element)
continue
if not is_visible and ui_element in ui_menu:
ui_menu.remove(ui_element)
continue
def deinit_ui(self):
self.__marker_window.close_window()
super().deinit_ui()
def recent_events(self, events):
super().recent_events(events)
frame = events.get("frame")
should_animate = True
state = self.__marker_window.window_state
if not frame:
return
self.__marker_window.update_state()
if not self.is_active:
# If the plugin is not active, just return
return
if self.marker_mode == SingleMarkerMode.MANUAL:
assert isinstance(
state, MarkerWindowStateClosed
), "In manual mode, window should be closed at all times."
if isinstance(state, MarkerWindowStateClosed):
if self.marker_mode != SingleMarkerMode.MANUAL:
# This state should be unreachable, since there is an early return if the plugin is inactive
assert not self.is_active
return
elif isinstance(state, MarkerWindowStateOpened):
assert self.is_active # Sanity check
assert self.marker_mode != SingleMarkerMode.MANUAL
pass # Continue with processing the frame
else:
raise UnhandledMarkerWindowStateError(state)
# Always save pupil positions
self.pupil_list.extend(events["pupil"])
gray_img = frame.gray
# Update the marker
ref_marker, stop_marker = self.__detect_ref_marker_and_stop_marker(gray_img)
self.__auto_stop_tracker.process_markers(stop_marker)
# Stop if autostop condition is satisfied
if self.__auto_stop_tracker.should_stop:
self._signal_should_stop(mode=self.current_mode)
# Signal marker window controller that a marker was detected (for feedback)
self.__marker_window.is_marker_detected = ref_marker is not None
should_save_ref_marker = False
if isinstance(state, MarkerWindowStateClosed):
should_save_ref_marker = self.marker_mode == SingleMarkerMode.MANUAL
elif isinstance(state, MarkerWindowStateIdle):
self.__marker_window.show_marker(
self._FIXED_MARKER_POSITION, should_animate=should_animate
)
elif isinstance(state, MarkerWindowStateAnimatingInMarker):
pass # No-op
elif isinstance(state, MarkerWindowStateShowingMarker):
assert self.marker_mode != SingleMarkerMode.MANUAL
should_save_ref_marker = True
elif isinstance(state, MarkerWindowStateAnimatingOutMarker):
pass # No-op
else:
raise UnhandledMarkerWindowStateError(state)
if should_save_ref_marker and ref_marker is not None and stop_marker is None:
ref = {}
ref["norm_pos"] = ref_marker["norm_pos"]
ref["screen_pos"] = ref_marker["img_pos"]
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# Update UI
self.__marker_window.draw_window()
self.status_text = self._FIXED_MARKER_POSITION if self.is_active else None
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if not self.is_active:
return
for marker in self.__previously_detected_markers:
# draw the largest ellipse of all detected markers
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0))
if len(self.__previously_detected_markers) > 1:
draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON)
# draw indicator on the stop marker(s)
if marker["marker_type"] == "Stop":
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self._STOP_MARKER_FRAMES_NEEDED_TO_STOP,
)
indicator = (
[e[0]]
+ pts[self.__auto_stop_tracker.detected_count :].tolist()
+ [e[0]]
)
draw_polyline(
indicator, color=RGBA(8.0, 0.1, 0.1, 0.8), line_type=gl.GL_POLYGON
)
### Internal
def _perform_start(self):
if not self.g_pool.capture.online:
logger.error(
f"{self.current_mode.label} requiers world capture video input."
)
return
self.__auto_stop_tracker.reset()
super()._perform_start()
if self.marker_mode != SingleMarkerMode.MANUAL:
is_fullscreen = self.marker_mode == SingleMarkerMode.FULL_SCREEN
self.__marker_window.open_window(
title=self.current_mode.label,
monitor_name=self.selected_monitor_name,
is_fullscreen=is_fullscreen,
)
def _perform_stop(self):
self.__marker_window.close_window()
super()._perform_stop()
### Private
def _on_window_did_close(self):
self._signal_should_stop(mode=self.current_mode)
def __detect_ref_marker_and_stop_marker(
self, gray_img
) -> T.Tuple[T.Optional[dict], T.Optional[dict]]:
markers = self.__circle_tracker.update(gray_img)
ref_marker = None
stop_marker = None
# Check if there are more than one markers
if len(markers) > 1:
logger.warning(
f"{len(markers)} markers detected. Please remove all the other markers"
)
for marker in markers:
if marker["marker_type"] == "Ref":
ref_marker = marker
if marker["marker_type"] == "Stop":
stop_marker = marker
self.__previously_detected_markers = [
m for m in [ref_marker, stop_marker] if m is not None
]
return ref_marker, stop_marker
class My_Manual_Marker_Calibration(Calibration_Plugin):
"""
CircleTracker looks for proper markers
Using at least 9 positions/points within the FOV
Ref detector will direct one to good positions with audio cues
Calibration only collects data at the good positions
"""
def __init__(self, g_pool):
super().__init__(g_pool)
self.pos = None
self.smooth_pos = 0.0, 0.0
self.smooth_vel = 0.0
self.sample_site = (-2, -2)
self.counter = 0
self.counter_max = 30
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.menu = None
self.ts_file = None
self.ts_filename = []
self.circle_tracker = CircleTracker()
self.markers = []
self.base_dir = []
#self.base_dir = '~/Desktop/agos_3d_calibration'
def init_ui(self):
super().init_ui()
self.menu.label = "Manual Calibration"
self.menu.append(
ui.Info_Text("Calibrate gaze parameters using a handheld marker."))
def start(self):
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Startingo {}".format(self.mode_pretty))
self.active = True
self.ref_list = []
self.pupil_list = []
self.ts_filename = os.path.join(self.base_dir, f"marker_center.csv")
print('BASE DIR: ' + self.ts_filename)
self.ts_file = open(self.ts_filename, 'a+')
def stop(self):
audio.say("Stopping {}".format(self.mode_pretty))
logger.info("Stoppingo {}".format(self.mode_pretty))
self.ts_file.close()
self.screen_marker_state = 0
self.active = False
self.smooth_pos = 0.0, 0.0
# self.close_window()
self.button.status_text = ""
if self.mode == "calibration":
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == "accuracy_test":
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
with open(self.ts_filename, 'a+') as self.ts_file:
self.ts_file.close()
def on_notify(self, notification):
if notification.get("subject") == 'recording.started':
self.base_dir = os.path.join(notification.get("rec_path"),
'3d_calibration')
os.makedirs(self.base_dir)
"""
Reacts to notifications:
``calibration.should_start``: Starts the calibration procedure
``calibration.should_stop``: Stops the calibration procedure
Emits notifications:
``calibration.started``: Calibration procedure started
``calibration.stopped``: Calibration procedure stopped
``calibration.marker_found``: Steady marker found
``calibration.marker_moved_too_quickly``: Marker moved too quickly
``calibration.marker_sample_completed``: Enough data points sampled
"""
super().on_notify(notification)
def recent_events(self, events):
"""
gets called once every frame.
reference positon need to be published to shared_pos
if no reference was found, publish 0,0
"""
frame = events.get("frame")
if self.active and frame:
gray_img = frame.gray
time_frame = self.g_pool.get_timestamp()
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
self.stop_marker_found = False
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]["img_pos"]
e = self.markers[0]["ellipses"]
self.pos = self.markers[0]["norm_pos"]
with open(self.ts_filename, 'a+') as self.ts_file:
self.ts_file.write(str(time_frame) + ",")
#self.ts_file.write(str(marker_pos[0]) + ',' + str(marker_pos[1]) + "\n")
self.ts_file.write(
str(e[0][0][0]) + ',' + str(e[0][0][1]) + ',' +
str(e[1][1][0]) + ',' + str(e[1][1][1]) + ',' +
str(e[0][2]) + "\n")
#self.ts_file.write(str(e[0]) + "\n")
# Check if there are stop markers
for marker in self.markers:
if marker["marker_type"] == "Stop":
self.auto_stop += 1
self.stop_marker_found = True
break
else:
self.pos = None # indicate that no reference is detected
if self.stop_marker_found is False:
self.auto_stop = 0
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".
format(len(self.markers)))
# tracking logic
if len(self.markers) and not self.stop_marker_found:
# start counter if ref is resting in place and not at last sample site
# calculate smoothed manhattan velocity
smoother = 0.3
smooth_pos = np.array(self.smooth_pos)
pos = np.array(self.pos)
new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
smooth_vel_vec = new_smooth_pos - smooth_pos
smooth_pos = new_smooth_pos
self.smooth_pos = list(smooth_pos)
# manhattan distance for velocity
new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
self.smooth_vel = self.smooth_vel + smoother * (
new_vel - self.smooth_vel)
# distance to last sampled site
sample_ref_dist = smooth_pos - np.array(self.sample_site)
sample_ref_dist = abs(sample_ref_dist[0]) + abs(
sample_ref_dist[1])
# start counter if ref is resting in place and not at last sample site
if self.counter <= 0:
if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
self.sample_site = self.smooth_pos
audio.beep()
logger.debug(
"Steady marker found. Starting to sample {} datapoints"
.format(self.counter_max))
self.notify_all({
"subject":
"calibration.marker_found",
"timestamp":
self.g_pool.get_timestamp(),
"record":
True,
})
self.counter = self.counter_max
if self.counter > 0:
if self.smooth_vel > 0.01:
audio.tink()
logger.warning(
"Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again."
.format(self.counter_max - self.counter))
self.notify_all({
"subject":
"calibration.marker_moved_too_quickly",
"timestamp":
self.g_pool.get_timestamp(),
"record":
True,
})
self.counter = 0
else:
self.counter -= 1
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
if self.counter <= 0:
# last sample before counter done and moving on
audio.tink()
logger.debug(
"Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
.format(self.counter_max))
self.notify_all({
"subject":
"calibration.marker_sample_completed",
"timestamp":
self.g_pool.get_timestamp(),
"record":
True,
})
# Always save pupil positions
self.pupil_list.extend(events["pupil"])
if self.counter:
if len(self.markers):
self.button.status_text = "Sampling Gaze Data"
else:
self.button.status_text = "Marker Lost"
else:
self.button.status_text = "Looking for Marker"
# Stop if autostop condition is satisfied:
if self.auto_stop >= self.auto_stop_max:
self.auto_stop = 0
self.stop()
else:
pass
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active:
draw_points_norm([self.smooth_pos],
size=15,
color=RGBA(1.0, 1.0, 0.0, 0.5))
if self.active and len(self.markers):
# draw the largest ellipse of all detected markers
for marker in self.markers:
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0))
if len(self.markers) > 1:
draw_polyline(pts,
1,
RGBA(1.0, 0.0, 0.0, 0.5),
line_type=GL_POLYGON)
# draw indicator on the first detected marker
if self.counter and self.markers[0]["marker_type"] == "Ref":
e = self.markers[0]["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self.counter_max,
)
indicator = [e[0]] + pts[self.counter:].tolist()[::-1] + [e[0]]
draw_polyline(indicator,
color=RGBA(0.1, 0.5, 0.7, 0.8),
line_type=GL_POLYGON)
# draw indicator on the stop marker(s)
if self.auto_stop:
for marker in self.markers:
if marker["marker_type"] == "Stop":
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self.auto_stop_max,
)
indicator = [e[0]] + pts[self.auto_stop:].tolist() + [
e[0]
]
draw_polyline(
indicator,
color=RGBA(8.0, 0.1, 0.1, 0.8),
line_type=GL_POLYGON,
)
else:
pass
def deinit_ui(self):
"""gets called when the plugin get terminated.
This happens either voluntarily or forced.
if you have an atb bar or glfw window destroy it here.
"""
if self.active:
self.stop()
super().deinit_ui()
class Screen_Marker_Calibration(Calibration_Plugin):
"""Calibrate using a marker on your screen
We use a ring detector that moves across the screen to 9 sites
Points are collected at sites - not between
"""
def __init__(
self,
g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.sample_duration = sample_duration # number of frames to sample per site
self.lead_in = 25 # frames of marker shown before starting to sample
self.lead_out = 5 # frames of markers shown after sampling is donw
self.active_site = None
self.sites = []
self.display_pos = -1.0, -1.0
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.monitor_idx = monitor_idx
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.menu.label = "Screen Marker Calibration"
def get_monitors_idx_list():
monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
return range(len(monitors)), monitors
if self.monitor_idx not in get_monitors_idx_list()[0]:
logger.warning(
"Monitor at index %s no longer availalbe using default"
% self.monitor_idx
)
self.monitor_idx = 0
self.menu.append(
ui.Info_Text("Calibrate gaze parameters using a screen based animation.")
)
self.menu.append(
ui.Selector(
"monitor_idx",
self,
selection_getter=get_monitors_idx_list,
label="Monitor",
)
)
self.menu.append(ui.Switch("fullscreen", self, label="Use fullscreen"))
self.menu.append(
ui.Slider(
"marker_scale", self, step=0.1, min=0.5, max=2.0, label="Marker size"
)
)
self.menu.append(
ui.Slider(
"sample_duration",
self,
step=1,
min=10,
max=100,
label="Sample duration",
)
)
def start(self):
if not self.g_pool.capture.online:
logger.error(
"{} requiers world capture video input.".format(self.mode_pretty)
)
return
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
if self.g_pool.detection_mapping_mode == "3d":
if self.mode == "calibration":
self.sites = [
(0.5, 0.5),
(0.0, 1.0),
(1.0, 1.0),
(1.0, 0.0),
(0.0, 0.0),
]
else:
self.sites = [(0.25, 0.5), (0.5, 0.25), (0.75, 0.5), (0.5, 0.75)]
else:
if self.mode == "calibration":
self.sites = [
(0.25, 0.5),
(0, 0.5),
(0.0, 1.0),
(0.5, 1.0),
(1.0, 1.0),
(1.0, 0.5),
(1.0, 0.0),
(0.5, 0.0),
(0.0, 0.0),
(0.75, 0.5),
]
else:
self.sites = [
(0.5, 0.5),
(0.25, 0.25),
(0.25, 0.75),
(0.75, 0.75),
(0.75, 0.25),
]
self.active_site = self.sites.pop(0)
self.active = True
self.ref_list = []
self.pupil_list = []
self.clicks_to_close = 5
self.open_window(self.mode_pretty)
def open_window(self, title="new_window"):
if not self._window:
if self.fullscreen:
try:
monitor = glfwGetMonitors()[self.monitor_idx]
except:
logger.warning(
"Monitor at index %s no longer availalbe using default"
% self.monitor_idx
)
self.monitor_idx = 0
monitor = glfwGetMonitors()[self.monitor_idx]
width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(
monitor
)
else:
monitor = None
width, height = 640, 360
self._window = glfwCreateWindow(
width, height, title, monitor=monitor, share=glfwGetCurrentContext()
)
if not self.fullscreen:
glfwSetWindowPos(
self._window,
self.window_position_default[0],
self.window_position_default[1],
)
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
# Register callbacks
glfwSetFramebufferSizeCallback(self._window, on_resize)
glfwSetKeyCallback(self._window, self.on_window_key)
glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)
on_resize(self._window, *glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
glfwMakeContextCurrent(active_window)
def on_window_key(self, window, key, scancode, action, mods):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
self.clicks_to_close = 0
def on_window_mouse_button(self, window, button, action, mods):
if action == GLFW_PRESS:
self.clicks_to_close -= 1
def stop(self):
# TODO: redundancy between all gaze mappers -> might be moved to parent class
audio.say("Stopping {}".format(self.mode_pretty))
logger.info("Stopping {}".format(self.mode_pretty))
self.smooth_pos = 0, 0
self.counter = 0
self.close_window()
self.active = False
self.button.status_text = ""
if self.mode == "calibration":
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == "accuracy_test":
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def close_window(self):
if self._window:
# enable mouse display
active_window = glfwGetCurrentContext()
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL)
glfwDestroyWindow(self._window)
self._window = None
glfwMakeContextCurrent(active_window)
def recent_events(self, events):
frame = events.get("frame")
if self.active and frame:
gray_img = frame.gray
if self.clicks_to_close <= 0:
self.stop()
return
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
# Screen marker takes only Ref marker
self.markers = [
marker for marker in self.markers if marker["marker_type"] == "Ref"
]
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]["img_pos"]
self.pos = self.markers[0]["norm_pos"]
else:
self.pos = None # indicate that no reference is detected
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".format(
len(self.markers)
)
)
# only save a valid ref position if within sample window of calibration routine
on_position = (
self.lead_in
< self.screen_marker_state
< (self.lead_in + self.sample_duration)
)
if on_position and len(self.markers):
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# Always save pupil positions
self.pupil_list.extend(events["pupil"])
if on_position and len(self.markers) and events.get("fixations", []):
fixation_boost = 5
self.screen_marker_state = min(
self.sample_duration + self.lead_in,
self.screen_marker_state + fixation_boost,
)
# Animate the screen marker
if (
self.screen_marker_state
< self.sample_duration + self.lead_in + self.lead_out
):
if len(self.markers) or not on_position:
self.screen_marker_state += 1
else:
self.screen_marker_state = 0
if not self.sites:
self.stop()
return
self.active_site = self.sites.pop(0)
logger.debug(
"Moving screen marker to site at {} {}".format(*self.active_site)
)
# use np.arrays for per element wise math
self.display_pos = np.array(self.active_site)
self.on_position = on_position
self.button.status_text = "{}".format(self.active_site)
if self._window:
self.gl_display_in_window()
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
# debug mode within world will show green ellipses around detected ellipses
if self.active:
for marker in self.markers:
e = marker["ellipses"][-1] # outermost ellipse
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0))
if len(self.markers) > 1:
draw_polyline(
pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON
)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = getHDPIFactor(self._window)
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = (
map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)),
map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)),
)
alpha = interp_fn(
self.screen_marker_state,
0.0,
1.0,
float(self.sample_duration + self.lead_in + self.lead_out),
float(self.lead_in),
float(self.sample_duration + self.lead_in),
)
r2 = 2 * r
draw_points(
[screen_pos], size=60 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.9
)
draw_points(
[screen_pos], size=38 * r2, color=RGBA(1.0, 1.0, 1.0, alpha), sharpness=0.8
)
draw_points(
[screen_pos], size=19 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.55
)
# some feedback on the detection state
color = (
RGBA(0.0, 0.8, 0.0, alpha)
if len(self.markers) and self.on_position
else RGBA(0.8, 0.0, 0.0, alpha)
)
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch {} more times to cancel {}.".format(
self.clicks_to_close, self.mode_pretty
),
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def get_init_dict(self):
d = {}
d["fullscreen"] = self.fullscreen
d["marker_scale"] = self.marker_scale
d["monitor_idx"] = self.monitor_idx
return d
def deinit_ui(self):
"""gets called when the plugin get terminated.
either voluntarily or forced.
"""
if self.active:
self.stop()
if self._window:
self.close_window()
super().deinit_ui()
class Participant_Driven_Screen_Marker_Calibration(Calibration_Plugin):
"""
Calibrate using on screen markers.
We use a ring detector that moves across the screen to 9 sites
Points are collected at sites - not between
Points are collected after space key is pressed
"""
def __init__(
self, g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40,
monitor_idx=1
):
super().__init__(g_pool)
self.detected = False
self.space_key_was_pressed = False
self.screen_marker_state = 0.
self.sample_duration = sample_duration # number of frames to sample per site
self.fixation_boost = sample_duration/2.
self.lead_in = 25 #frames of marker shown before starting to sample
self.lead_out = 5 #frames of markers shown after sampling is donw
self.monitor_idx = monitor_idx
self.active_site = None
self.sites = []
self.display_pos = -1., -1.
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.button = None
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center')
# UI Platform tweaks
if system() == 'Linux':
self.window_position_default = (0, 0)
elif system() == 'Windows':
self.window_position_default = (8, 31)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.menu.label = "Participant Driven Screen Marker Calibration"
self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
self.menu.append(ui.Info_Text("Calibrate gaze parameters pressing space key for each marker."))
self.menu.append(ui.Selector('monitor_idx',self,selection = range(len(self.monitor_names)),labels=self.monitor_names,label='Monitor'))
self.menu.append(ui.Switch('fullscreen',self,label='Use fullscreen'))
self.menu.append(ui.Slider('marker_scale',self,step=0.1,min=0.5,max=2.0,label='Marker size'))
self.menu.append(ui.Slider('sample_duration',self,step=1,min=10,max=100,label='Sample duration'))
def start(self):
if not self.g_pool.capture.online:
logger.error("{} requires world capture video input.".format(self.mode_pretty))
return
super().start()
logger.info("Starting {}".format(self.mode_pretty))
if self.mode == 'calibration':
self.sites = [(.5, .5), (.25, .5), (0, .5), (1., .5), (.75, .5),
(.5, 1.), (.25, 1.), (0., 1.), (1., 1.), (.75, 1.),
(.25, .0), (1., 0.), (.5, 0.), (0., 0.), (.75, .0)
]
shuffle(self.sites)
else:
self.sites = [(.5, .5), (.25, .25), (.25, .75), (.75, .75), (.75, .25)]
self.active_site = self.sites.pop(0)
self.active = True
self.ref_list = []
self.pupil_list = []
self.clicks_to_close = 5
self.open_window(self.mode_pretty)
def open_window(self,title='new_window'):
if not self._window:
if self.fullscreen:
monitor = glfwGetMonitors()[self.monitor_idx]
width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(monitor)
else:
monitor = None
width,height= 640,360
self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext())
if not self.fullscreen:
glfwSetWindowPos(self._window, self.window_position_default[0], self.window_position_default[1])
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
# Register callbacks
glfwSetFramebufferSizeCallback(self._window, on_resize)
glfwSetKeyCallback(self._window, self.on_window_key)
glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)
on_resize(self._window, *glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
glfwMakeContextCurrent(active_window)
def on_window_key(self,window, key, scancode, action, mods):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
self.clicks_to_close = 0
if key == GLFW_KEY_SPACE:
self.space_key_was_pressed = True
def on_window_mouse_button(self,window,button, action, mods):
if action ==GLFW_PRESS:
self.clicks_to_close -=1
def stop(self):
# TODO: redundancy between all gaze mappers -> might be moved to parent class
logger.info("Stopping {}".format(self.mode_pretty))
self.smooth_pos = 0, 0
self.counter = 0
self.close_window()
self.active = False
self.button.status_text = ''
if self.mode == 'calibration':
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == 'accuracy_test':
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def close_window(self):
if self._window:
# enable mouse display
active_window = glfwGetCurrentContext();
glfwSetInputMode(self._window,GLFW_CURSOR,GLFW_CURSOR_NORMAL)
glfwDestroyWindow(self._window)
self._window = None
glfwMakeContextCurrent(active_window)
def recent_events(self, events):
frame = events.get('frame')
if self.active and frame:
recent_pupil_positions = events['pupil_positions']
gray_img = frame.gray
if self.clicks_to_close <=0:
self.stop()
return
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
# Screen marker takes only Ref marker
self.markers = [marker for marker in self.markers if marker['marker_type'] == 'Ref']
if len(self.markers) > 0:
self.detected = True
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]['img_pos']
self.pos = self.markers[0]['norm_pos']
else:
self.detected = False
self.pos = None # indicate that no reference is detected
if len(self.markers) > 1:
logger.warning("{} markers detected. Please remove all the other markers".format(len(self.markers)))
# only save a valid ref position if within sample window of calibraiton routine
on_position = self.lead_in < self.screen_marker_state < (self.lead_in+self.sample_duration)
if on_position and self.detected and self.space_key_was_pressed:
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# always save pupil positions
for p_pt in recent_pupil_positions:
if p_pt['confidence'] > self.pupil_confidence_threshold:
self.pupil_list.append(p_pt)
if on_position and self.detected and events.get('fixations', []) and self.space_key_was_pressed:
self.screen_marker_state = min(
self.sample_duration+self.lead_in,
self.screen_marker_state+self.fixation_boost)
# Animate the screen marker
if self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out:
if (self.detected and self.space_key_was_pressed) or not on_position:
self.screen_marker_state += 1
else:
self.space_key_was_pressed = False
self.screen_marker_state = 0
if not self.sites:
self.stop()
return
self.active_site = self.sites.pop(0)
logger.debug("Moving screen marker to site at {} {}".format(*self.active_site))
# use np.arrays for per element wise math
self.display_pos = np.array(self.active_site)
self.on_position = on_position
self.button.status_text = '{} / {}'.format(self.active_site, 15)
if self._window:
self.gl_display_in_window()
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
# debug mode within world will show green ellipses around detected ellipses
if self.active and self.detected:
for marker in self.markers:
e = marker['ellipses'][-1] # outermost ellipse
pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])),
(int(e[1][0]/2), int(e[1][1]/2)),
int(e[-1]), 0, 360, 15)
draw_polyline(pts, 1, RGBA(0.,1.,0.,1.))
if len(self.markers) > 1:
draw_polyline(pts, 1, RGBA(1., 0., 0., .5), line_type=gl.GL_POLYGON)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = glfwGetFramebufferSize(self._window)[0]/glfwGetWindowSize(self._window)[0]
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value,in_range=(0,1),out_range=(0,1)):
ratio = (out_range[1]-out_range[0])/(in_range[1]-in_range[0])
return (value-in_range[0])*ratio+out_range[0]
pad = 90 * r
screen_pos = map_value(self.display_pos[0],out_range=(pad,p_window_size[0]-pad)),map_value(self.display_pos[1],out_range=(p_window_size[1]-pad,pad))
alpha = interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
r2 = 2 * r
draw_points([screen_pos], size=60*r2, color=RGBA(0., 0., 0., alpha), sharpness=0.9)
draw_points([screen_pos], size=38*r2, color=RGBA(1., 1., 1., alpha), sharpness=0.8)
draw_points([screen_pos], size=19*r2, color=RGBA(0., 0., 0., alpha), sharpness=0.55)
# some feedback on the detection state and button pressing
if self.detected and self.on_position and self.space_key_was_pressed:
color = RGBA(.8,.8,0., alpha)
else:
if self.detected:
color = RGBA(0.,.8,0., alpha)
else:
color = RGBA(.8,0.,0., alpha)
draw_points([screen_pos],size=3*r2,color=color,sharpness=0.5)
if self.clicks_to_close <5:
self.glfont.set_size(int(p_window_size[0]/30.))
self.glfont.draw_text(p_window_size[0]/2.,p_window_size[1]/4.,'Touch {} more times to cancel {}.'.format(self.clicks_to_close, self.mode_pretty))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def get_init_dict(self):
d = {}
d['fullscreen'] = self.fullscreen
d['marker_scale'] = self.marker_scale
d['sample_duration'] = self.sample_duration
d['monitor_idx'] = self.monitor_idx
return d
def deinit_ui(self):
"""gets called when the plugin get terminated.
either voluntarily or forced.
"""
if self.active:
self.stop()
if self._window:
self.close_window()
super().deinit_ui()
def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20):
# ipc setup
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url)
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from time import sleep
from video_capture import File_Source, EndofVideoFileError
from circle_detector import CircleTracker
try:
src = File_Source(Empty(), source_path, timed_playback=False)
frame = src.get_frame()
logger.info('Starting calibration marker detection...')
frame_count = src.get_frame_count()
queue = []
circle_tracker = CircleTracker()
while True:
while process_pipe.new_data:
topic, n = process_pipe.recv()
if topic == 'terminate':
process_pipe.send(topic='exception',
payload={"reason": "User terminated."})
logger.debug("Process terminated")
sleep(1.0)
return
progress = 100. * frame.index / frame_count
markers = [
m for m in circle_tracker.update(frame.gray)
if m['marker_type'] == 'Ref'
]
if len(markers):
ref = {
"norm_pos": markers[0]['norm_pos'],
"screen_pos": markers[0]['img_pos'],
"timestamp": frame.timestamp,
'index': frame.index
}
queue.append((progress, ref))
else:
queue.append((progress, None))
if len(queue) > batch_size:
# dequeue batch
data = queue[:batch_size]
del queue[:batch_size]
process_pipe.send(topic='progress', payload={'data': data})
frame = src.get_frame()
except EndofVideoFileError:
process_pipe.send(topic='progress', payload={'data': queue})
process_pipe.send(topic='finished', payload={})
logger.debug("Process finished")
except Exception:
import traceback
process_pipe.send(topic='exception',
payload={'reason': traceback.format_exc()})
logger.debug("Process raised Exception")
sleep(1.0)
class Screen_Marker_Calibration(Calibration_Plugin):
"""Calibrate using a marker on your screen
We use a ring detector that moves across the screen to 9 sites
Points are collected at sites - not between
"""
def __init__(
self,
g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.sample_duration = sample_duration # number of frames to sample per site
self.lead_in = 25 # frames of marker shown before starting to sample
self.lead_out = 5 # frames of markers shown after sampling is donw
self.active_site = None
self.sites = []
self.display_pos = -1.0, -1.0
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.monitor_idx = monitor_idx
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.menu.label = "Screen Marker Calibration"
def get_monitors_idx_list():
monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
return range(len(monitors)), monitors
if self.monitor_idx not in get_monitors_idx_list()[0]:
logger.warning(
"Monitor at index %s no longer availalbe using default" %
self.monitor_idx)
self.monitor_idx = 0
self.menu.append(
ui.Info_Text(
"Calibrate gaze parameters using a screen based animation."))
self.menu.append(
ui.Selector(
"monitor_idx",
self,
selection_getter=get_monitors_idx_list,
label="Monitor",
))
self.menu.append(ui.Switch("fullscreen", self, label="Use fullscreen"))
self.menu.append(
ui.Slider("marker_scale",
self,
step=0.1,
min=0.5,
max=2.0,
label="Marker size"))
self.menu.append(
ui.Slider(
"sample_duration",
self,
step=1,
min=10,
max=100,
label="Sample duration",
))
def start(self):
if not self.g_pool.capture.online:
logger.error("{} requiers world capture video input.".format(
self.mode_pretty))
return
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
if self.g_pool.detection_mapping_mode == "3d":
if self.mode == "calibration":
self.sites = [
(0.5, 0.5),
(0.0, 1.0),
(1.0, 1.0),
(1.0, 0.0),
(0.0, 0.0),
]
else:
self.sites = [(0.25, 0.5), (0.5, 0.25), (0.75, 0.5),
(0.5, 0.75)]
else:
if self.mode == "calibration":
self.sites = [
(0.25, 0.5),
(0, 0.5),
(0.0, 1.0),
(0.5, 1.0),
(1.0, 1.0),
(1.0, 0.5),
(1.0, 0.0),
(0.5, 0.0),
(0.0, 0.0),
(0.75, 0.5),
]
else:
self.sites = [
(0.5, 0.5),
(0.25, 0.25),
(0.25, 0.75),
(0.75, 0.75),
(0.75, 0.25),
]
self.active_site = self.sites.pop(0)
self.active = True
self.ref_list = []
self.pupil_list = []
self.clicks_to_close = 5
self.open_window(self.mode_pretty)
def open_window(self, title="new_window"):
if not self._window:
if self.fullscreen:
try:
monitor = glfwGetMonitors()[self.monitor_idx]
except:
logger.warning(
"Monitor at index %s no longer availalbe using default"
% self.monitor_idx)
self.monitor_idx = 0
monitor = glfwGetMonitors()[self.monitor_idx]
width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(
monitor)
else:
monitor = None
width, height = 640, 360
self._window = glfwCreateWindow(width,
height,
title,
monitor=monitor,
share=glfwGetCurrentContext())
if not self.fullscreen:
glfwSetWindowPos(
self._window,
self.window_position_default[0],
self.window_position_default[1],
)
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
# Register callbacks
glfwSetFramebufferSizeCallback(self._window, on_resize)
glfwSetKeyCallback(self._window, self.on_window_key)
glfwSetMouseButtonCallback(self._window,
self.on_window_mouse_button)
on_resize(self._window, *glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
glfwMakeContextCurrent(active_window)
def on_window_key(self, window, key, scancode, action, mods):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
self.clicks_to_close = 0
def on_window_mouse_button(self, window, button, action, mods):
if action == GLFW_PRESS:
self.clicks_to_close -= 1
def stop(self):
# TODO: redundancy between all gaze mappers -> might be moved to parent class
audio.say("Stopping {}".format(self.mode_pretty))
logger.info("Stopping {}".format(self.mode_pretty))
self.smooth_pos = 0, 0
self.counter = 0
self.close_window()
self.active = False
self.button.status_text = ""
if self.mode == "calibration":
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == "accuracy_test":
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def close_window(self):
if self._window:
# enable mouse display
active_window = glfwGetCurrentContext()
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL)
glfwDestroyWindow(self._window)
self._window = None
glfwMakeContextCurrent(active_window)
def recent_events(self, events):
frame = events.get("frame")
if self.active and frame:
gray_img = frame.gray
if self.clicks_to_close <= 0:
self.stop()
return
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
# Screen marker takes only Ref marker
self.markers = [
marker for marker in self.markers
if marker["marker_type"] == "Ref"
]
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]["img_pos"]
self.pos = self.markers[0]["norm_pos"]
else:
self.pos = None # indicate that no reference is detected
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".
format(len(self.markers)))
# only save a valid ref position if within sample window of calibration routine
on_position = (self.lead_in < self.screen_marker_state <
(self.lead_in + self.sample_duration))
if on_position and len(self.markers):
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# Always save pupil positions
self.pupil_list.extend(events["pupil"])
if on_position and len(self.markers) and events.get(
"fixations", []):
fixation_boost = 5
self.screen_marker_state = min(
self.sample_duration + self.lead_in,
self.screen_marker_state + fixation_boost,
)
# Animate the screen marker
if (self.screen_marker_state <
self.sample_duration + self.lead_in + self.lead_out):
if len(self.markers) or not on_position:
self.screen_marker_state += 1
else:
self.screen_marker_state = 0
if not self.sites:
self.stop()
return
self.active_site = self.sites.pop(0)
logger.debug("Moving screen marker to site at {} {}".format(
*self.active_site))
# use np.arrays for per element wise math
self.display_pos = np.array(self.active_site)
self.on_position = on_position
self.button.status_text = "{}".format(self.active_site)
if self._window:
self.gl_display_in_window()
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
# debug mode within world will show green ellipses around detected ellipses
if self.active:
for marker in self.markers:
e = marker["ellipses"][-1] # outermost ellipse
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0))
if len(self.markers) > 1:
draw_polyline(pts,
1,
RGBA(1.0, 0.0, 0.0, 0.5),
line_type=gl.GL_POLYGON)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = getHDPIFactor(self._window)
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = (
map_value(self.display_pos[0],
out_range=(pad, p_window_size[0] - pad)),
map_value(self.display_pos[1],
out_range=(p_window_size[1] - pad, pad)),
)
alpha = interp_fn(
self.screen_marker_state,
0.0,
1.0,
float(self.sample_duration + self.lead_in + self.lead_out),
float(self.lead_in),
float(self.sample_duration + self.lead_in),
)
r2 = 2 * r
draw_points([screen_pos],
size=60 * r2,
color=RGBA(0.0, 0.0, 0.0, alpha),
sharpness=0.9)
draw_points([screen_pos],
size=38 * r2,
color=RGBA(1.0, 1.0, 1.0, alpha),
sharpness=0.8)
draw_points([screen_pos],
size=19 * r2,
color=RGBA(0.0, 0.0, 0.0, alpha),
sharpness=0.55)
# some feedback on the detection state
color = (RGBA(0.0, 0.8, 0.0, alpha) if len(self.markers)
and self.on_position else RGBA(0.8, 0.0, 0.0, alpha))
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch {} more times to cancel {}.".format(
self.clicks_to_close, self.mode_pretty),
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def get_init_dict(self):
d = {}
d["fullscreen"] = self.fullscreen
d["marker_scale"] = self.marker_scale
d["monitor_idx"] = self.monitor_idx
return d
def deinit_ui(self):
"""gets called when the plugin get terminated.
either voluntarily or forced.
"""
if self.active:
self.stop()
if self._window:
self.close_window()
super().deinit_ui()
class ScreenMarkerChoreographyPlugin(
MonitorSelectionMixin, CalibrationChoreographyPlugin
):
"""Calibrate using a marker on your screen
We use a ring detector that moves across the screen to 9 sites
Points are collected at sites - not between
"""
label = "Screen Marker Calibration"
@classmethod
def selection_label(cls) -> str:
return "Screen Marker"
@classmethod
def selection_order(cls) -> float:
return 1.0
@staticmethod
def get_list_of_markers_to_show(mode: ChoreographyMode) -> list:
if ChoreographyMode.CALIBRATION == mode:
return [(0.5, 0.5), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0), (0.0, 0.0)]
if ChoreographyMode.VALIDATION == mode:
return [(0.5, 1.0), (1.0, 0.5), (0.5, 0.0), (0.0, 0.5)]
raise ValueError(f"Unknown mode {mode}")
def __init__(
self,
g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40,
monitor_name=None,
**kwargs,
):
super().__init__(g_pool, **kwargs)
# Public properties
self.selected_monitor_name = monitor_name
self.is_fullscreen = fullscreen
self.sample_duration = sample_duration
# Private properties
self.__current_list_of_markers_to_show = []
self.__currently_shown_marker_position = None
self.__ref_count_for_current_marker_position = 0
self.__previously_detected_markers = []
self.__circle_tracker = CircleTracker()
self.__marker_window = MarkerWindowController(marker_scale=marker_scale)
self.__marker_window.add_observer(
"on_window_did_close", self._on_window_did_close
)
def get_init_dict(self):
d = {}
d["fullscreen"] = self.is_fullscreen
d["marker_scale"] = self.__marker_window.marker_scale
d["monitor_name"] = self.selected_monitor_name
return d
### Public - Plugin
@classmethod
def _choreography_description_text(cls) -> str:
return "Calibrate gaze parameters using a screen based animation."
def _init_custom_menu_ui_elements(self) -> list:
self.__ui_selector_monitor_name = ui.Selector(
"selected_monitor_name",
self,
label="Monitor",
labels=self.currently_connected_monitor_names(),
selection=self.currently_connected_monitor_names(),
)
self.__ui_switch_is_fullscreen = ui.Switch(
"is_fullscreen", self, label="Use fullscreen"
)
self.__ui_slider_marker_scale = ui.Slider(
"marker_scale",
self.__marker_window,
label="Marker size",
min=0.5,
max=2.0,
step=0.1,
)
self.__ui_slider_sample_duration = ui.Slider(
"sample_duration", self, label="Sample duration", min=10, max=100, step=1
)
return [
self.__ui_selector_monitor_name,
self.__ui_switch_is_fullscreen,
self.__ui_slider_marker_scale,
self.__ui_slider_sample_duration,
]
def deinit_ui(self):
self.__marker_window.close_window()
super().deinit_ui()
def recent_events(self, events):
super().recent_events(events)
frame = events.get("frame")
state = self.__marker_window.window_state
should_animate = True
if not frame:
return
self.__marker_window.update_state()
if isinstance(state, MarkerWindowStateClosed):
return
elif isinstance(state, MarkerWindowStateOpened):
assert self.is_active # Sanity check
pass # Continue with processing the frame
else:
raise UnhandledMarkerWindowStateError(state)
# Always save pupil positions
self.pupil_list.extend(events["pupil"])
# Detect reference circle marker
detected_marker = self.__detect_reference_circle_marker(frame.gray)
# Signal marker window controller that a marker was detected (for feedback)
self.__marker_window.is_marker_detected = detected_marker is not None
if isinstance(state, MarkerWindowStateIdle):
assert self.__currently_shown_marker_position is None # Sanity check
if self.__current_list_of_markers_to_show:
self.__currently_shown_marker_position = (
self.__current_list_of_markers_to_show.pop(0)
)
logger.debug(
f"Moving screen marker to site at {self.__currently_shown_marker_position}"
)
self.__marker_window.show_marker(
marker_position=self.__currently_shown_marker_position,
should_animate=should_animate,
)
return
else:
# No more markers to show; stop calibration choreography.
self._signal_should_stop(mode=self.current_mode)
return
if isinstance(state, MarkerWindowStateAnimatingInMarker):
assert self.__currently_shown_marker_position is not None # Sanity check
pass # No-op
elif isinstance(state, MarkerWindowStateShowingMarker):
assert self.__currently_shown_marker_position is not None # Sanity check
if detected_marker is not None:
ref = {}
ref["norm_pos"] = detected_marker["norm_pos"]
ref["screen_pos"] = detected_marker["img_pos"]
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
should_move_to_next_marker = len(self.ref_list) == self.sample_duration * (
self.__ref_count_for_current_marker_position + 1
)
if should_move_to_next_marker:
# Finished collecting samples for current active site
self.__currently_shown_marker_position = None
self.__ref_count_for_current_marker_position += 1
self.__marker_window.hide_marker(should_animate=should_animate)
elif isinstance(state, MarkerWindowStateAnimatingOutMarker):
assert self.__currently_shown_marker_position is None # Sanity check
pass # No-op
else:
raise UnhandledMarkerWindowStateError(state)
# Update UI
self.__marker_window.draw_window()
self.status_text = self.__currently_shown_marker_position
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
# debug mode within world will show green ellipses around detected ellipses
if not self.is_active:
return
markers = self.__previously_detected_markers
for marker in markers:
e = marker["ellipses"][-1] # outermost ellipse
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0))
if len(markers) > 1:
draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON)
### Internal
def _perform_start(self):
if not self.g_pool.capture.online:
logger.error(
f"{self.current_mode.label} requiers world capture video input."
)
return
self.__current_list_of_markers_to_show = self.get_list_of_markers_to_show(
mode=self.current_mode,
)
self.__currently_shown_marker_position = None
self.__ref_count_for_current_marker_position = 0
super()._perform_start()
self.__marker_window.open_window(
title=self.current_mode.label,
monitor_name=self.selected_monitor_name,
is_fullscreen=self.is_fullscreen,
)
def _perform_stop(self):
self.__marker_window.close_window()
super()._perform_stop()
### Private
def _on_window_did_close(self):
self._signal_should_stop(mode=self.current_mode)
def __detect_reference_circle_marker(self, gray_img):
# Detect all circular markers
circle_markers = self.__circle_tracker.update(gray_img)
# Only keep Ref markers
circle_markers = [
marker for marker in circle_markers if marker["marker_type"] == "Ref"
]
# Store detected Ref markers for debugging/visualization
self.__previously_detected_markers = circle_markers
if len(circle_markers) == 0:
return None
elif len(circle_markers) == 1:
return circle_markers[0]
else:
logger.warning(
f"{len(circle_markers)} markers detected. Please remove all the other markers"
)
return circle_markers[0]
class Single_Marker_Calibration(Calibration_Plugin):
"""Calibrate using a single marker.
Move your head for example in a spiral motion while gazing
at the marker to quickly sample a wide range gaze angles.
"""
def __init__(self,
g_pool,
fullscreen=True,
marker_scale=1.0,
sample_duration=40):
super().__init__(g_pool)
self.screen_marker_state = 0.
self.lead_in = 25 # frames of marker shown before starting to sample
self.display_pos = (.5, .5)
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.fullscreen = fullscreen
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font('opensans', get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align='center')
# UI Platform tweaks
if system() == 'Linux':
self.window_position_default = (0, 0)
elif system() == 'Windows':
self.window_position_default = (8, 31)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.monitor_idx = 0
self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
#primary_monitor = glfwGetPrimaryMonitor()
self.menu.append(
ui.Info_Text(
"Calibrate gaze parameters using a single gae targets and active head movements."
))
self.menu.append(
ui.Selector('monitor_idx',
self,
selection=range(len(self.monitor_names)),
labels=self.monitor_names,
label='Monitor'))
self.menu.append(ui.Switch('fullscreen', self, label='Use fullscreen'))
self.menu.append(
ui.Slider('marker_scale',
self,
step=0.1,
min=0.5,
max=2.0,
label='Marker size'))
def start(self):
if not self.g_pool.capture.online:
logger.error("Calibration required world capture video input.")
return
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
self.active = True
self.ref_list = []
self.pupil_list = []
self.clicks_to_close = 5
self.open_window(self.mode_pretty)
def open_window(self, title='new_window'):
if not self._window:
if self.fullscreen:
monitor = glfwGetMonitors()[self.monitor_idx]
width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(
monitor)
else:
monitor = None
width, height = 640, 360
self._window = glfwCreateWindow(width,
height,
title,
monitor=monitor,
share=glfwGetCurrentContext())
if not self.fullscreen:
glfwSetWindowPos(self._window, self.window_position_default[0],
self.window_position_default[1])
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
# Register callbacks
glfwSetFramebufferSizeCallback(self._window, on_resize)
glfwSetKeyCallback(self._window, self.on_window_key)
glfwSetMouseButtonCallback(self._window,
self.on_window_mouse_button)
on_resize(self._window, *glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
glfwMakeContextCurrent(active_window)
def on_window_key(self, window, key, scancode, action, mods):
if action == GLFW_PRESS:
if self.mode == 'calibration':
target_key = GLFW_KEY_C
else:
target_key = GLFW_KEY_T
if key == GLFW_KEY_ESCAPE or key == target_key:
self.clicks_to_close = 0
def on_window_mouse_button(self, window, button, action, mods):
if action == GLFW_PRESS:
self.clicks_to_close -= 1
def stop(self):
# TODO: redundancy between all gaze mappers -> might be moved to parent class
audio.say("Stopping {}".format(self.mode_pretty))
logger.info('Stopping {}'.format(self.mode_pretty))
self.smooth_pos = 0, 0
self.counter = 0
self.close_window()
self.active = False
self.button.status_text = ''
if self.mode == 'calibration':
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == 'accuracy_test':
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def close_window(self):
if self._window:
# enable mouse display
active_window = glfwGetCurrentContext()
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL)
glfwDestroyWindow(self._window)
self._window = None
glfwMakeContextCurrent(active_window)
def recent_events(self, events):
frame = events.get('frame')
if self.active and frame:
recent_pupil_positions = events['pupil_positions']
gray_img = frame.gray
if self.clicks_to_close <= 0:
self.stop()
return
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
# Screen marker takes only Ref marker
self.markers = [
marker for marker in self.markers
if marker['marker_type'] == 'Ref'
]
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]['img_pos']
self.pos = self.markers[0]['norm_pos']
else:
self.pos = None # indicate that no reference is detected
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".
format(len(self.markers)))
# only save a valid ref position if within sample window of calibraiton routine
on_position = self.lead_in < self.screen_marker_state
if on_position and len(self.markers):
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# always save pupil positions
for p_pt in recent_pupil_positions:
if p_pt['confidence'] > self.pupil_confidence_threshold:
self.pupil_list.append(p_pt)
# Animate the screen marker
if len(self.markers) or not on_position:
self.screen_marker_state += 1
# use np.arrays for per element wise math
self.on_position = on_position
if self._window:
self.gl_display_in_window()
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
# debug mode within world will show green ellipses around detected ellipses
if self.active:
for marker in self.markers:
e = marker['ellipses'][-1] # outermost ellipse
pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]), 0, 360, 15)
draw_polyline(pts, 1, RGBA(0., 1., 0., 1.))
if len(self.markers) > 1:
draw_polyline(pts,
1,
RGBA(1., 0., 0., .5),
line_type=gl.GL_POLYGON)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = glfwGetFramebufferSize(
self._window)[0] / glfwGetWindowSize(self._window)[0]
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = map_value(
self.display_pos[0],
out_range=(pad, p_window_size[0] - pad)), map_value(
self.display_pos[1], out_range=(p_window_size[1] - pad, pad))
alpha = 1.0 #interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
r2 = 2 * r
draw_points([screen_pos],
size=60 * r2,
color=RGBA(0., 0., 0., alpha),
sharpness=0.9)
draw_points([screen_pos],
size=38 * r2,
color=RGBA(1., 1., 1., alpha),
sharpness=0.8)
draw_points([screen_pos],
size=19 * r2,
color=RGBA(0., 0., 0., alpha),
sharpness=0.55)
# some feedback on the detection state
color = RGBA(0., .8, 0., alpha) if len(
self.markers) and self.on_position else RGBA(0.8, 0., 0., alpha)
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.))
self.glfont.draw_text(
p_window_size[0] / 2., p_window_size[1] / 4.,
'Touch {} more times to cancel calibration.'.format(
self.clicks_to_close))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def get_init_dict(self):
d = {}
d['fullscreen'] = self.fullscreen
d['marker_scale'] = self.marker_scale
return d
def deinit_ui(self):
"""gets called when the plugin get terminated.
either voluntarily or forced.
"""
if self.active:
self.stop()
if self._window:
self.close_window()
super().deinit_ui()
class Single_Marker_Calibration(Calibration_Plugin):
"""Calibrate using a single marker.
Move your head for example in a spiral motion while gazing
at the marker to quickly sample a wide range gaze angles.
"""
def __init__(
self,
g_pool,
marker_mode="Full screen",
marker_scale=1.0,
sample_duration=40,
monitor_idx=0,
):
super().__init__(g_pool)
self.screen_marker_state = 0.0
self.lead_in = 25 # frames of marker shown before starting to sample
self.display_pos = (0.5, 0.5)
self.on_position = False
self.pos = None
self.marker_scale = marker_scale
self._window = None
self.menu = None
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.monitor_idx = monitor_idx
self.marker_mode = marker_mode
self.clicks_to_close = 5
self.glfont = fontstash.Context()
self.glfont.add_font("opensans", get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align="center")
# UI Platform tweaks
if system() == "Linux":
self.window_position_default = (0, 0)
elif system() == "Windows":
self.window_position_default = (8, 90)
else:
self.window_position_default = (0, 0)
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
def get_monitors_idx_list():
monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
return range(len(monitors)), monitors
if self.monitor_idx not in get_monitors_idx_list()[0]:
logger.warning(
"Monitor at index %s no longer availalbe using default" % idx
)
self.monitor_idx = 0
self.menu.append(
ui.Info_Text(
"Calibrate using a single marker. Gaze at the center of the marker and move your head (e.g. in a slow spiral movement). This calibration method enables you to quickly sample a wide range of gaze angles and cover a large range of your FOV."
)
)
self.menu.append(
ui.Selector(
"marker_mode",
self,
selection=["Full screen", "Window", "Manual"],
label="Marker display mode",
)
)
self.menu.append(
ui.Selector(
"monitor_idx",
self,
selection_getter=get_monitors_idx_list,
label="Monitor",
)
)
self.menu.append(
ui.Slider(
"marker_scale", self, step=0.1, min=0.5, max=2.0, label="Marker size"
)
)
def start(self):
if not self.g_pool.capture.online:
logger.error("This calibration requires world capture video input.")
return
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
self.active = True
self.ref_list = []
self.pupil_list = []
self.clicks_to_close = 5
if self.marker_mode != "Manual":
self.open_window(self.mode_pretty)
def open_window(self, title="new_window"):
if not self._window:
if self.marker_mode == "Full screen":
try:
monitor = glfwGetMonitors()[self.monitor_idx]
except:
logger.warning(
"Monitor at index %s no longer availalbe using default" % idx
)
self.monitor_idx = 0
monitor = glfwGetMonitors()[self.monitor_idx]
width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(
monitor
)
else:
monitor = None
width, height = 640, 360
self._window = glfwCreateWindow(
width, height, title, monitor=monitor, share=glfwGetCurrentContext()
)
if self.marker_mode == "Window":
glfwSetWindowPos(
self._window,
self.window_position_default[0],
self.window_position_default[1],
)
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
# Register callbacks
glfwSetFramebufferSizeCallback(self._window, on_resize)
glfwSetKeyCallback(self._window, self.on_window_key)
glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)
on_resize(self._window, *glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
glfwMakeContextCurrent(active_window)
def on_window_key(self, window, key, scancode, action, mods):
if action == GLFW_PRESS:
if self.mode == "calibration":
target_key = GLFW_KEY_C
else:
target_key = GLFW_KEY_T
if key == GLFW_KEY_ESCAPE or key == target_key:
self.clicks_to_close = 0
def on_window_mouse_button(self, window, button, action, mods):
if action == GLFW_PRESS:
self.clicks_to_close -= 1
def stop(self):
# TODO: redundancy between all gaze mappers -> might be moved to parent class
audio.say("Stopping {}".format(self.mode_pretty))
logger.info("Stopping {}".format(self.mode_pretty))
self.smooth_pos = 0, 0
self.counter = 0
self.close_window()
self.active = False
self.button.status_text = ""
if self.mode == "calibration":
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == "accuracy_test":
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def close_window(self):
if self._window:
# enable mouse display
active_window = glfwGetCurrentContext()
glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL)
glfwDestroyWindow(self._window)
self._window = None
glfwMakeContextCurrent(active_window)
def recent_events(self, events):
frame = events.get("frame")
if self.active and frame:
gray_img = frame.gray
if self.clicks_to_close <= 0:
self.stop()
return
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
self.stop_marker_found = False
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]["img_pos"]
self.pos = self.markers[0]["norm_pos"]
# Check if there are stop markers
for marker in self.markers:
if marker["marker_type"] == "Stop":
self.auto_stop += 1
self.stop_marker_found = True
break
else:
self.pos = None # indicate that no reference is detected
if self.stop_marker_found is False:
self.auto_stop = 0
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".format(
len(self.markers)
)
)
# only save a valid ref position if within sample window of calibraiton routine
on_position = self.lead_in < self.screen_marker_state
if on_position and len(self.markers) and not self.stop_marker_found:
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
# always save pupil positions
self.pupil_list.extend(events["pupil"])
# Animate the screen marker
if len(self.markers) or not on_position:
self.screen_marker_state += 1
# Stop if autostop condition is satisfied:
if self.auto_stop >= self.auto_stop_max:
self.auto_stop = 0
self.stop()
# use np.arrays for per element wise math
self.on_position = on_position
if self._window:
self.gl_display_in_window()
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active:
# draw the largest ellipse of all detected markers
for marker in self.markers:
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0))
if len(self.markers) > 1:
draw_polyline(
pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON
)
# draw indicator on the stop marker(s)
if self.auto_stop:
for marker in self.markers:
if marker["marker_type"] == "Stop":
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self.auto_stop_max,
)
indicator = [e[0]] + pts[self.auto_stop :].tolist() + [e[0]]
draw_polyline(
indicator,
color=RGBA(8.0, 0.1, 0.1, 0.8),
line_type=gl.GL_POLYGON,
)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = getHDPIFactor(self._window)
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = (
map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)),
map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)),
)
alpha = (
1.0
) # interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
r2 = 2 * r
draw_points(
[screen_pos], size=60 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.9
)
draw_points(
[screen_pos], size=38 * r2, color=RGBA(1.0, 1.0, 1.0, alpha), sharpness=0.8
)
draw_points(
[screen_pos], size=19 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.55
)
# some feedback on the detection state
color = (
RGBA(0.0, 0.8, 0.0, alpha)
if len(self.markers) and self.on_position
else RGBA(0.8, 0.0, 0.0, alpha)
)
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch {} more times to cancel calibration.".format(
self.clicks_to_close
),
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def get_init_dict(self):
d = {}
d["marker_mode"] = self.marker_mode
d["marker_scale"] = self.marker_scale
d["monitor_idx"] = self.monitor_idx
return d
def deinit_ui(self):
"""gets called when the plugin get terminated.
either voluntarily or forced.
"""
if self.active:
self.stop()
if self._window:
self.close_window()
super().deinit_ui()
class Manual_Marker_Calibration(Calibration_Plugin):
"""
CircleTracker looks for proper markers
Using at least 9 positions/points within the FOV
Ref detector will direct one to good positions with audio cues
Calibration only collects data at the good positions
"""
def __init__(self, g_pool):
super().__init__(g_pool)
self.pos = None
self.smooth_pos = 0., 0.
self.smooth_vel = 0.
self.sample_site = (-2, -2)
self.counter = 0
self.counter_max = 30
self.stop_marker_found = False
self.auto_stop = 0
self.auto_stop_max = 30
self.menu = None
self.circle_tracker = CircleTracker()
self.markers = []
def init_ui(self):
super().init_ui()
self.menu.label = "Manual Calibration"
self.menu.append(
ui.Info_Text("Calibrate gaze parameters using a handheld marker."))
def start(self):
super().start()
audio.say("Starting {}".format(self.mode_pretty))
logger.info("Starting {}".format(self.mode_pretty))
self.active = True
self.ref_list = []
self.pupil_list = []
def stop(self):
audio.say("Stopping {}".format(self.mode_pretty))
logger.info('Stopping {}'.format(self.mode_pretty))
self.screen_marker_state = 0
self.active = False
# self.close_window()
self.button.status_text = ''
if self.mode == 'calibration':
finish_calibration(self.g_pool, self.pupil_list, self.ref_list)
elif self.mode == 'accuracy_test':
self.finish_accuracy_test(self.pupil_list, self.ref_list)
super().stop()
def on_notify(self, notification):
'''
Reacts to notifications:
``calibration.should_start``: Starts the calibration procedure
``calibration.should_stop``: Stops the calibration procedure
Emits notifications:
``calibration.started``: Calibration procedure started
``calibration.stopped``: Calibration procedure stopped
``calibration.marker_found``: Steady marker found
``calibration.marker_moved_too_quickly``: Marker moved too quickly
``calibration.marker_sample_completed``: Enough data points sampled
'''
super().on_notify(notification)
def recent_events(self, events):
"""
gets called once every frame.
reference positon need to be published to shared_pos
if no reference was found, publish 0,0
"""
frame = events.get('frame')
if self.active and frame:
recent_pupil_positions = events['pupil_positions']
gray_img = frame.gray
# Update the marker
self.markers = self.circle_tracker.update(gray_img)
self.stop_marker_found = False
if len(self.markers):
# Set the pos to be the center of the first detected marker
marker_pos = self.markers[0]['img_pos']
self.pos = self.markers[0]['norm_pos']
# Check if there are stop markers
for marker in self.markers:
if marker['marker_type'] == 'Stop':
self.auto_stop += 1
self.stop_marker_found = True
break
else:
self.pos = None # indicate that no reference is detected
if self.stop_marker_found is False:
self.auto_stop = 0
# Check if there are more than one markers
if len(self.markers) > 1:
audio.tink()
logger.warning(
"{} markers detected. Please remove all the other markers".
format(len(self.markers)))
# tracking logic
if len(self.markers) and not self.stop_marker_found:
# start counter if ref is resting in place and not at last sample site
# calculate smoothed manhattan velocity
smoother = 0.3
smooth_pos = np.array(self.smooth_pos)
pos = np.array(self.pos)
new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
smooth_vel_vec = new_smooth_pos - smooth_pos
smooth_pos = new_smooth_pos
self.smooth_pos = list(smooth_pos)
#manhattan distance for velocity
new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
self.smooth_vel = self.smooth_vel + smoother * (
new_vel - self.smooth_vel)
#distance to last sampled site
sample_ref_dist = smooth_pos - np.array(self.sample_site)
sample_ref_dist = abs(sample_ref_dist[0]) + abs(
sample_ref_dist[1])
# start counter if ref is resting in place and not at last sample site
if self.counter <= 0:
if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
self.sample_site = self.smooth_pos
audio.beep()
logger.debug(
"Steady marker found. Starting to sample {} datapoints"
.format(self.counter_max))
self.notify_all({
'subject':
'calibration.marker_found',
'timestamp':
self.g_pool.get_timestamp(),
'record':
True
})
self.counter = self.counter_max
if self.counter > 0:
if self.smooth_vel > 0.01:
audio.tink()
logger.warning(
"Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again."
.format(self.counter_max - self.counter))
self.notify_all({
'subject':
'calibration.marker_moved_too_quickly',
'timestamp':
self.g_pool.get_timestamp(),
'record':
True
})
self.counter = 0
else:
self.counter -= 1
ref = {}
ref["norm_pos"] = self.pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
self.ref_list.append(ref)
if events.get('fixations', []):
self.counter -= 5
if self.counter <= 0:
#last sample before counter done and moving on
audio.tink()
logger.debug(
"Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
.format(self.counter_max))
self.notify_all({
'subject':
'calibration.marker_sample_completed',
'timestamp':
self.g_pool.get_timestamp(),
'record':
True
})
# Always save pupil positions
for p_pt in recent_pupil_positions:
if p_pt['confidence'] > self.pupil_confidence_threshold:
self.pupil_list.append(p_pt)
if self.counter:
if len(self.markers):
self.button.status_text = 'Sampling Gaze Data'
else:
self.button.status_text = 'Marker Lost'
else:
self.button.status_text = 'Looking for Marker'
# Stop if autostop condition is satisfied:
if self.auto_stop >= self.auto_stop_max:
self.auto_stop = 0
self.stop()
else:
pass
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active:
draw_points_norm([self.smooth_pos],
size=15,
color=RGBA(1., 1., 0., .5))
if self.active and len(self.markers):
# draw the largest ellipse of all detected markers
for marker in self.markers:
e = marker['ellipses'][-1]
pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]), 0, 360, 15)
draw_polyline(pts, color=RGBA(0., 1., 0, 1.))
if len(self.markers) > 1:
draw_polyline(pts,
1,
RGBA(1., 0., 0., .5),
line_type=GL_POLYGON)
# draw indicator on the first detected marker
if self.counter and self.markers[0]['marker_type'] == 'Ref':
e = self.markers[0]['ellipses'][-1]
pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]), 0, 360,
360 // self.counter_max)
indicator = [e[0]] + pts[self.counter:].tolist()[::-1] + [e[0]]
draw_polyline(indicator,
color=RGBA(0.1, .5, .7, .8),
line_type=GL_POLYGON)
# draw indicator on the stop marker(s)
if self.auto_stop:
for marker in self.markers:
if marker['marker_type'] == 'Stop':
e = marker['ellipses'][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]),
0, 360, 360 // self.auto_stop_max)
indicator = [e[0]] + pts[self.auto_stop:].tolist() + [
e[0]
]
draw_polyline(indicator,
color=RGBA(8., 0.1, 0.1, .8),
line_type=GL_POLYGON)
else:
pass
def deinit_ui(self):
"""gets called when the plugin get terminated.
This happens either voluntarily or forced.
if you have an atb bar or glfw window destroy it here.
"""
if self.active:
self.stop()
super().deinit_ui()
def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20):
# ipc setup
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url)
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from time import sleep
from video_capture import init_playback_source, EndofVideoError
from circle_detector import CircleTracker
try:
src = init_playback_source(Empty(), source_path, timing=None)
frame = src.get_frame()
logger.info("Starting calibration marker detection...")
frame_count = src.get_frame_count()
queue = []
circle_tracker = CircleTracker()
while True:
while process_pipe.new_data:
topic, n = process_pipe.recv()
if topic == "terminate":
process_pipe.send({
"topic": "exception",
"reason": "User terminated."
})
logger.debug("Process terminated")
sleep(1.0)
return
progress = 100.0 * frame.index / frame_count
markers = [
m for m in circle_tracker.update(frame.gray)
if m["marker_type"] == "Ref"
]
if len(markers):
ref = {
"norm_pos": markers[0]["norm_pos"],
"screen_pos": markers[0]["img_pos"],
"timestamp": frame.timestamp,
"index_range":
tuple(range(frame.index - 5, frame.index + 5)),
"index": frame.index,
}
queue.append((progress, ref))
else:
queue.append((progress, None))
if len(queue) > batch_size:
# dequeue batch
data = queue[:batch_size]
del queue[:batch_size]
process_pipe.send({"topic": "progress", "data": data})
frame = src.get_frame()
except EndofVideoError:
process_pipe.send({"topic": "progress", "data": queue})
process_pipe.send({"topic": "finished"})
logger.debug("Process finished")
except Exception:
import traceback
process_pipe.send({
"topic": "exception",
"reason": traceback.format_exc()
})
logger.debug("Process raised Exception")
sleep(1.0)
