class Single_Marker_Calibration(Calibration_Plugin): """Calibrate using a single marker. Move your head for example in a spiral motion while gazing at the marker to quickly sample a wide range gaze angles. """ def __init__( self, g_pool, marker_mode="Full screen", marker_scale=1.0, sample_duration=40, monitor_idx=0, ): super().__init__(g_pool) self.screen_marker_state = 0.0 self.lead_in = 25 # frames of marker shown before starting to sample self.display_pos = (0.5, 0.5) self.on_position = False self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.stop_marker_found = False self.auto_stop = 0 self.auto_stop_max = 30 self.monitor_idx = monitor_idx self.marker_mode = marker_mode self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font("opensans", get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0)) self.glfont.set_align_string(v_align="center") # UI Platform tweaks if system() == "Linux": self.window_position_default = (0, 0) elif system() == "Windows": self.window_position_default = (8, 90) else: self.window_position_default = (0, 0) self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()] def get_monitors_idx_list(): monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()] return range(len(monitors)), monitors if self.monitor_idx not in get_monitors_idx_list()[0]: logger.warning( "Monitor at index %s no longer availalbe using default" % idx) self.monitor_idx = 0 self.menu.append( ui.Info_Text( "Calibrate using a single marker. Gaze at the center of the marker and move your head (e.g. in a slow spiral movement). This calibration method enables you to quickly sample a wide range of gaze angles and cover a large range of your FOV." )) self.menu.append( ui.Selector( "marker_mode", self, selection=["Full screen", "Window", "Manual"], label="Marker display mode", )) self.menu.append( ui.Selector( "monitor_idx", self, selection_getter=get_monitors_idx_list, label="Monitor", )) self.menu.append( ui.Slider("marker_scale", self, step=0.1, min=0.5, max=2.0, label="Marker size")) def start(self): if not self.g_pool.capture.online: logger.error( "This calibration requires world capture video input.") return super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) self.active = True self.ref_list = [] self.pupil_list = [] self.clicks_to_close = 5 if self.marker_mode != "Manual": self.open_window(self.mode_pretty) def open_window(self, title="new_window"): if not self._window: if self.marker_mode == "Full screen": try: monitor = glfwGetMonitors()[self.monitor_idx] except Exception: logger.warning( "Monitor at index %s no longer availalbe using default" % idx) self.monitor_idx = 0 monitor = glfwGetMonitors()[self.monitor_idx] width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode( monitor) else: monitor = None width, height = 640, 360 self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext()) if self.marker_mode == "Window": glfwSetWindowPos( self._window, self.window_position_default[0], self.window_position_default[1], ) glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN) # Register callbacks glfwSetFramebufferSizeCallback(self._window, on_resize) glfwSetKeyCallback(self._window, self.on_window_key) glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button) on_resize(self._window, *glfwGetFramebufferSize(self._window)) # gl_state settings active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) def on_window_key(self, window, key, scancode, action, mods): if action == GLFW_PRESS: if self.mode == "calibration": target_key = GLFW_KEY_C else: target_key = GLFW_KEY_T if key == GLFW_KEY_ESCAPE or key == target_key: self.clicks_to_close = 0 def on_window_mouse_button(self, window, button, action, mods): if action == GLFW_PRESS: self.clicks_to_close -= 1 def stop(self): # TODO: redundancy between all gaze mappers -> might be moved to parent class audio.say("Stopping {}".format(self.mode_pretty)) logger.info("Stopping {}".format(self.mode_pretty)) self.smooth_pos = 0, 0 self.counter = 0 self.close_window() self.active = False self.button.status_text = "" if self.mode == "calibration": finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == "accuracy_test": self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def close_window(self): if self._window: # enable mouse display active_window = glfwGetCurrentContext() glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL) glfwDestroyWindow(self._window) self._window = None glfwMakeContextCurrent(active_window) def recent_events(self, events): frame = events.get("frame") if self.active and frame: gray_img = frame.gray if self.clicks_to_close <= 0: self.stop() return # Update the marker self.markers = self.circle_tracker.update(gray_img) self.stop_marker_found = False if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]["img_pos"] self.pos = self.markers[0]["norm_pos"] # Check if there are stop markers for marker in self.markers: if marker["marker_type"] == "Stop": self.auto_stop += 1 self.stop_marker_found = True break else: self.pos = None # indicate that no reference is detected if self.stop_marker_found is False: self.auto_stop = 0 # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers". format(len(self.markers))) # only save a valid ref position if within sample window of calibraiton routine on_position = self.lead_in < self.screen_marker_state if on_position and len( self.markers) and not self.stop_marker_found: ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # always save pupil positions self.pupil_list.extend(events["pupil"]) # Animate the screen marker if len(self.markers) or not on_position: self.screen_marker_state += 1 # Stop if autostop condition is satisfied: if self.auto_stop >= self.auto_stop_max: self.auto_stop = 0 self.stop() # use np.arrays for per element wise math self.on_position = on_position if self._window: self.gl_display_in_window() def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: # draw the largest ellipse of all detected markers for marker in self.markers: e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON) # draw indicator on the stop marker(s) if self.auto_stop: for marker in self.markers: if marker["marker_type"] == "Stop": e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.auto_stop_max, ) indicator = [e[0]] + pts[self.auto_stop:].tolist() + [ e[0] ] draw_polyline( indicator, color=RGBA(8.0, 0.1, 0.1, 0.8), line_type=gl.GL_POLYGON, ) def gl_display_in_window(self): active_window = glfwGetCurrentContext() if glfwWindowShouldClose(self._window): self.close_window() return glfwMakeContextCurrent(self._window) clear_gl_screen() hdpi_factor = getHDPIFactor(self._window) r = self.marker_scale * hdpi_factor gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() p_window_size = glfwGetFramebufferSize(self._window) gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1) # Switch back to Model View Matrix gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() def map_value(value, in_range=(0, 1), out_range=(0, 1)): ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0]) return (value - in_range[0]) * ratio + out_range[0] pad = 90 * r screen_pos = ( map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)), map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)), ) alpha = ( 1.0 ) # interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in)) r2 = 2 * r draw_points([screen_pos], size=60 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.9) draw_points([screen_pos], size=38 * r2, color=RGBA(1.0, 1.0, 1.0, alpha), sharpness=0.8) draw_points([screen_pos], size=19 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.55) # some feedback on the detection state color = (RGBA(0.0, 0.8, 0.0, alpha) if len(self.markers) and self.on_position else RGBA(0.8, 0.0, 0.0, alpha)) draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5) if self.clicks_to_close < 5: self.glfont.set_size(int(p_window_size[0] / 30.0)) self.glfont.draw_text( p_window_size[0] / 2.0, p_window_size[1] / 4.0, "Touch {} more times to cancel calibration.".format( self.clicks_to_close), ) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) def get_init_dict(self): d = {} d["marker_mode"] = self.marker_mode d["marker_scale"] = self.marker_scale d["monitor_idx"] = self.monitor_idx return d def deinit_ui(self): """gets called when the plugin get terminated. either voluntarily or forced. """ if self.active: self.stop() if self._window: self.close_window() super().deinit_ui()
class Manual_Marker_Calibration(Calibration_Plugin): """ CircleTracker looks for proper markers Using at least 9 positions/points within the FOV Ref detector will direct one to good positions with audio cues Calibration only collects data at the good positions """ def __init__(self, g_pool): super().__init__(g_pool) self.pos = None self.smooth_pos = 0.0, 0.0 self.smooth_vel = 0.0 self.sample_site = (-2, -2) self.counter = 0 self.counter_max = 30 self.stop_marker_found = False self.auto_stop = 0 self.auto_stop_max = 30 self.menu = None self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.menu.label = "Manual Calibration" self.menu.append( ui.Info_Text("Calibrate gaze parameters using a handheld marker.") ) def start(self): super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) self.active = True self.ref_list = [] self.pupil_list = [] def stop(self): audio.say("Stopping {}".format(self.mode_pretty)) logger.info("Stopping {}".format(self.mode_pretty)) self.screen_marker_state = 0 self.active = False self.smooth_pos = 0.0, 0.0 # self.close_window() self.button.status_text = "" if self.mode == "calibration": finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == "accuracy_test": self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def on_notify(self, notification): """ Reacts to notifications: ``calibration.should_start``: Starts the calibration procedure ``calibration.should_stop``: Stops the calibration procedure Emits notifications: ``calibration.started``: Calibration procedure started ``calibration.stopped``: Calibration procedure stopped ``calibration.marker_found``: Steady marker found ``calibration.marker_moved_too_quickly``: Marker moved too quickly ``calibration.marker_sample_completed``: Enough data points sampled """ super().on_notify(notification) def recent_events(self, events): """ gets called once every frame. reference positon need to be published to shared_pos if no reference was found, publish 0,0 """ frame = events.get("frame") if self.active and frame: gray_img = frame.gray # Update the marker self.markers = self.circle_tracker.update(gray_img) self.stop_marker_found = False if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]["img_pos"] self.pos = self.markers[0]["norm_pos"] # Check if there are stop markers for marker in self.markers: if marker["marker_type"] == "Stop": self.auto_stop += 1 self.stop_marker_found = True break else: self.pos = None # indicate that no reference is detected if self.stop_marker_found is False: self.auto_stop = 0 # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers".format( len(self.markers) ) ) # tracking logic if len(self.markers) and not self.stop_marker_found: # start counter if ref is resting in place and not at last sample site # calculate smoothed manhattan velocity smoother = 0.3 smooth_pos = np.array(self.smooth_pos) pos = np.array(self.pos) new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos) smooth_vel_vec = new_smooth_pos - smooth_pos smooth_pos = new_smooth_pos self.smooth_pos = list(smooth_pos) # manhattan distance for velocity new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1]) self.smooth_vel = self.smooth_vel + smoother * ( new_vel - self.smooth_vel ) # distance to last sampled site sample_ref_dist = smooth_pos - np.array(self.sample_site) sample_ref_dist = abs(sample_ref_dist[0]) + abs(sample_ref_dist[1]) # start counter if ref is resting in place and not at last sample site if self.counter <= 0: if self.smooth_vel < 0.01 and sample_ref_dist > 0.1: self.sample_site = self.smooth_pos audio.beep() logger.debug( "Steady marker found. Starting to sample {} datapoints".format( self.counter_max ) ) self.notify_all( { "subject": "calibration.marker_found", "timestamp": self.g_pool.get_timestamp(), "record": True, } ) self.counter = self.counter_max if self.counter > 0: if self.smooth_vel > 0.01: audio.tink() logger.warning( "Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again.".format( self.counter_max - self.counter ) ) self.notify_all( { "subject": "calibration.marker_moved_too_quickly", "timestamp": self.g_pool.get_timestamp(), "record": True, } ) self.counter = 0 else: self.counter -= 1 ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) if events.get("fixations", []): self.counter -= 5 if self.counter <= 0: # last sample before counter done and moving on audio.tink() logger.debug( "Sampled {} datapoints. Stopping to sample. Looking for steady marker again.".format( self.counter_max ) ) self.notify_all( { "subject": "calibration.marker_sample_completed", "timestamp": self.g_pool.get_timestamp(), "record": True, } ) # Always save pupil positions self.pupil_list.extend(events["pupil"]) if self.counter: if len(self.markers): self.button.status_text = "Sampling Gaze Data" else: self.button.status_text = "Marker Lost" else: self.button.status_text = "Looking for Marker" # Stop if autostop condition is satisfied: if self.auto_stop >= self.auto_stop_max: self.auto_stop = 0 self.stop() else: pass def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: draw_points_norm([self.smooth_pos], size=15, color=RGBA(1.0, 1.0, 0.0, 0.5)) if self.active and len(self.markers): # draw the largest ellipse of all detected markers for marker in self.markers: e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0)) if len(self.markers) > 1: draw_polyline( pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=GL_POLYGON ) # draw indicator on the first detected marker if self.counter and self.markers[0]["marker_type"] == "Ref": e = self.markers[0]["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.counter_max, ) indicator = [e[0]] + pts[self.counter :].tolist()[::-1] + [e[0]] draw_polyline( indicator, color=RGBA(0.1, 0.5, 0.7, 0.8), line_type=GL_POLYGON ) # draw indicator on the stop marker(s) if self.auto_stop: for marker in self.markers: if marker["marker_type"] == "Stop": e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.auto_stop_max, ) indicator = [e[0]] + pts[self.auto_stop :].tolist() + [e[0]] draw_polyline( indicator, color=RGBA(8.0, 0.1, 0.1, 0.8), line_type=GL_POLYGON, ) else: pass def deinit_ui(self): """gets called when the plugin get terminated. This happens either voluntarily or forced. if you have an atb bar or glfw window destroy it here. """ if self.active: self.stop() super().deinit_ui()
class SingleMarkerChoreographyPlugin( MonitorSelectionMixin, CalibrationChoreographyPlugin ): """Calibrate using a single marker. Move your head for example in a spiral motion while gazing at the marker to quickly sample a wide range gaze angles. """ label = "Single Marker Calibration" @classmethod def selection_label(cls) -> str: return "Single Marker" @classmethod def selection_order(cls) -> float: return 2.0 _STOP_MARKER_FRAMES_NEEDED_TO_STOP = 30 _FIXED_MARKER_POSITION = (0.5, 0.5) def __init__( self, g_pool, marker_mode=None, marker_scale=1.0, sample_duration=40, monitor_name=None, **kwargs, ): if marker_mode is None: marker_mode = SingleMarkerMode.MANUAL else: marker_mode = SingleMarkerMode.from_label(marker_mode) super().__init__(g_pool, **kwargs) # Public properties self.selected_monitor_name = monitor_name self.marker_mode = marker_mode # Private properties self.__previously_detected_markers = [] self.__circle_tracker = CircleTracker() self.__auto_stop_tracker = AutoStopTracker( markers_needed=self._STOP_MARKER_FRAMES_NEEDED_TO_STOP ) self.__marker_window = MarkerWindowController(marker_scale=marker_scale) self.__marker_window.add_observer( "on_window_did_close", self._on_window_did_close ) def get_init_dict(self): d = {} d["marker_mode"] = self.marker_mode.label d["marker_scale"] = self.__marker_window.marker_scale d["monitor_name"] = self.selected_monitor_name return d def cleanup(self): super().cleanup() @property def marker_mode(self) -> SingleMarkerMode: return self.__marker_mode @marker_mode.setter def marker_mode(self, value: SingleMarkerMode): self.__marker_mode = value self._ui_update_visibility_digital_marker_config() ### Public - Plugin @classmethod def _choreography_description_text(cls) -> str: return "Calibrate using a single marker. Gaze at the center of the marker and move your head (e.g. in a slow spiral movement). This calibration method enables you to quickly sample a wide range of gaze angles and cover a large range of your FOV." def _init_custom_menu_ui_elements(self) -> list: self.__ui_selector_marker_mode = ui.Selector( "marker_mode", self, label="Marker display mode", labels=[m.label for m in SingleMarkerMode.all_modes()], selection=SingleMarkerMode.all_modes(), ) # TODO: potential race condition through selection_getter. Should ensure # that current selection will always be present in the list returned by the # selection_getter. Highly unlikely though as this needs to happen between # having clicked the Selector and the next redraw. # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b self.__ui_selector_monitor_name = ui.Selector( "selected_monitor_name", self, label="Monitor", labels=self.currently_connected_monitor_names(), selection=self.currently_connected_monitor_names(), ) self.__ui_slider_marker_scale = ui.Slider( "marker_scale", self.__marker_window, label="Marker size", min=0.5, max=2.0, step=0.1, ) return [ self.__ui_selector_marker_mode, self.__ui_selector_monitor_name, self.__ui_slider_marker_scale, ] def init_ui(self): super().init_ui() # Save UI elements that are part of the digital marker config self.__ui_digital_marker_config_elements = [ self.__ui_selector_monitor_name, self.__ui_slider_marker_scale, ] # Save start index of the UI elements of digital marker config self.__ui_digital_marker_config_start_index = min( self.menu.elements.index(elem) for elem in self.__ui_digital_marker_config_elements ) self._ui_update_visibility_digital_marker_config() def _ui_update_visibility_digital_marker_config(self): try: ui_menu = self.menu ui_elements = self.__ui_digital_marker_config_elements start_index = self.__ui_digital_marker_config_start_index except AttributeError: return is_visible = self.marker_mode != SingleMarkerMode.MANUAL for i, ui_element in enumerate(ui_elements): index = start_index + i if is_visible and ui_element not in ui_menu: ui_menu.insert(index, ui_element) continue if not is_visible and ui_element in ui_menu: ui_menu.remove(ui_element) continue def deinit_ui(self): self.__marker_window.close_window() super().deinit_ui() def recent_events(self, events): super().recent_events(events) frame = events.get("frame") should_animate = True state = self.__marker_window.window_state if not frame: return self.__marker_window.update_state() if not self.is_active: # If the plugin is not active, just return return if self.marker_mode == SingleMarkerMode.MANUAL: assert isinstance( state, MarkerWindowStateClosed ), "In manual mode, window should be closed at all times." if isinstance(state, MarkerWindowStateClosed): if self.marker_mode != SingleMarkerMode.MANUAL: # This state should be unreachable, since there is an early return if the plugin is inactive assert not self.is_active return elif isinstance(state, MarkerWindowStateOpened): assert self.is_active # Sanity check assert self.marker_mode != SingleMarkerMode.MANUAL pass # Continue with processing the frame else: raise UnhandledMarkerWindowStateError(state) # Always save pupil positions self.pupil_list.extend(events["pupil"]) gray_img = frame.gray # Update the marker ref_marker, stop_marker = self.__detect_ref_marker_and_stop_marker(gray_img) self.__auto_stop_tracker.process_markers(stop_marker) # Stop if autostop condition is satisfied if self.__auto_stop_tracker.should_stop: self._signal_should_stop(mode=self.current_mode) # Signal marker window controller that a marker was detected (for feedback) self.__marker_window.is_marker_detected = ref_marker is not None should_save_ref_marker = False if isinstance(state, MarkerWindowStateClosed): should_save_ref_marker = self.marker_mode == SingleMarkerMode.MANUAL elif isinstance(state, MarkerWindowStateIdle): self.__marker_window.show_marker( self._FIXED_MARKER_POSITION, should_animate=should_animate ) elif isinstance(state, MarkerWindowStateAnimatingInMarker): pass # No-op elif isinstance(state, MarkerWindowStateShowingMarker): assert self.marker_mode != SingleMarkerMode.MANUAL should_save_ref_marker = True elif isinstance(state, MarkerWindowStateAnimatingOutMarker): pass # No-op else: raise UnhandledMarkerWindowStateError(state) if should_save_ref_marker and ref_marker is not None and stop_marker is None: ref = {} ref["norm_pos"] = ref_marker["norm_pos"] ref["screen_pos"] = ref_marker["img_pos"] ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # Update UI self.__marker_window.draw_window() self.status_text = self._FIXED_MARKER_POSITION if self.is_active else None def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if not self.is_active: return for marker in self.__previously_detected_markers: # draw the largest ellipse of all detected markers e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0)) if len(self.__previously_detected_markers) > 1: draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON) # draw indicator on the stop marker(s) if marker["marker_type"] == "Stop": e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self._STOP_MARKER_FRAMES_NEEDED_TO_STOP, ) indicator = ( [e[0]] + pts[self.__auto_stop_tracker.detected_count :].tolist() + [e[0]] ) draw_polyline( indicator, color=RGBA(8.0, 0.1, 0.1, 0.8), line_type=gl.GL_POLYGON ) ### Internal def _perform_start(self): if not self.g_pool.capture.online: logger.error( f"{self.current_mode.label} requiers world capture video input." ) return self.__auto_stop_tracker.reset() super()._perform_start() if self.marker_mode != SingleMarkerMode.MANUAL: is_fullscreen = self.marker_mode == SingleMarkerMode.FULL_SCREEN self.__marker_window.open_window( title=self.current_mode.label, monitor_name=self.selected_monitor_name, is_fullscreen=is_fullscreen, ) def _perform_stop(self): self.__marker_window.close_window() super()._perform_stop() ### Private def _on_window_did_close(self): self._signal_should_stop(mode=self.current_mode) def __detect_ref_marker_and_stop_marker( self, gray_img ) -> T.Tuple[T.Optional[dict], T.Optional[dict]]: markers = self.__circle_tracker.update(gray_img) ref_marker = None stop_marker = None # Check if there are more than one markers if len(markers) > 1: logger.warning( f"{len(markers)} markers detected. Please remove all the other markers" ) for marker in markers: if marker["marker_type"] == "Ref": ref_marker = marker if marker["marker_type"] == "Stop": stop_marker = marker self.__previously_detected_markers = [ m for m in [ref_marker, stop_marker] if m is not None ] return ref_marker, stop_marker
class My_Manual_Marker_Calibration(Calibration_Plugin): """ CircleTracker looks for proper markers Using at least 9 positions/points within the FOV Ref detector will direct one to good positions with audio cues Calibration only collects data at the good positions """ def __init__(self, g_pool): super().__init__(g_pool) self.pos = None self.smooth_pos = 0.0, 0.0 self.smooth_vel = 0.0 self.sample_site = (-2, -2) self.counter = 0 self.counter_max = 30 self.stop_marker_found = False self.auto_stop = 0 self.auto_stop_max = 30 self.menu = None self.ts_file = None self.ts_filename = [] self.circle_tracker = CircleTracker() self.markers = [] self.base_dir = [] #self.base_dir = '~/Desktop/agos_3d_calibration' def init_ui(self): super().init_ui() self.menu.label = "Manual Calibration" self.menu.append( ui.Info_Text("Calibrate gaze parameters using a handheld marker.")) def start(self): super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Startingo {}".format(self.mode_pretty)) self.active = True self.ref_list = [] self.pupil_list = [] self.ts_filename = os.path.join(self.base_dir, f"marker_center.csv") print('BASE DIR: ' + self.ts_filename) self.ts_file = open(self.ts_filename, 'a+') def stop(self): audio.say("Stopping {}".format(self.mode_pretty)) logger.info("Stoppingo {}".format(self.mode_pretty)) self.ts_file.close() self.screen_marker_state = 0 self.active = False self.smooth_pos = 0.0, 0.0 # self.close_window() self.button.status_text = "" if self.mode == "calibration": finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == "accuracy_test": self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() with open(self.ts_filename, 'a+') as self.ts_file: self.ts_file.close() def on_notify(self, notification): if notification.get("subject") == 'recording.started': self.base_dir = os.path.join(notification.get("rec_path"), '3d_calibration') os.makedirs(self.base_dir) """ Reacts to notifications: ``calibration.should_start``: Starts the calibration procedure ``calibration.should_stop``: Stops the calibration procedure Emits notifications: ``calibration.started``: Calibration procedure started ``calibration.stopped``: Calibration procedure stopped ``calibration.marker_found``: Steady marker found ``calibration.marker_moved_too_quickly``: Marker moved too quickly ``calibration.marker_sample_completed``: Enough data points sampled """ super().on_notify(notification) def recent_events(self, events): """ gets called once every frame. reference positon need to be published to shared_pos if no reference was found, publish 0,0 """ frame = events.get("frame") if self.active and frame: gray_img = frame.gray time_frame = self.g_pool.get_timestamp() # Update the marker self.markers = self.circle_tracker.update(gray_img) self.stop_marker_found = False if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]["img_pos"] e = self.markers[0]["ellipses"] self.pos = self.markers[0]["norm_pos"] with open(self.ts_filename, 'a+') as self.ts_file: self.ts_file.write(str(time_frame) + ",") #self.ts_file.write(str(marker_pos[0]) + ',' + str(marker_pos[1]) + "\n") self.ts_file.write( str(e[0][0][0]) + ',' + str(e[0][0][1]) + ',' + str(e[1][1][0]) + ',' + str(e[1][1][1]) + ',' + str(e[0][2]) + "\n") #self.ts_file.write(str(e[0]) + "\n") # Check if there are stop markers for marker in self.markers: if marker["marker_type"] == "Stop": self.auto_stop += 1 self.stop_marker_found = True break else: self.pos = None # indicate that no reference is detected if self.stop_marker_found is False: self.auto_stop = 0 # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers". format(len(self.markers))) # tracking logic if len(self.markers) and not self.stop_marker_found: # start counter if ref is resting in place and not at last sample site # calculate smoothed manhattan velocity smoother = 0.3 smooth_pos = np.array(self.smooth_pos) pos = np.array(self.pos) new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos) smooth_vel_vec = new_smooth_pos - smooth_pos smooth_pos = new_smooth_pos self.smooth_pos = list(smooth_pos) # manhattan distance for velocity new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1]) self.smooth_vel = self.smooth_vel + smoother * ( new_vel - self.smooth_vel) # distance to last sampled site sample_ref_dist = smooth_pos - np.array(self.sample_site) sample_ref_dist = abs(sample_ref_dist[0]) + abs( sample_ref_dist[1]) # start counter if ref is resting in place and not at last sample site if self.counter <= 0: if self.smooth_vel < 0.01 and sample_ref_dist > 0.1: self.sample_site = self.smooth_pos audio.beep() logger.debug( "Steady marker found. Starting to sample {} datapoints" .format(self.counter_max)) self.notify_all({ "subject": "calibration.marker_found", "timestamp": self.g_pool.get_timestamp(), "record": True, }) self.counter = self.counter_max if self.counter > 0: if self.smooth_vel > 0.01: audio.tink() logger.warning( "Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again." .format(self.counter_max - self.counter)) self.notify_all({ "subject": "calibration.marker_moved_too_quickly", "timestamp": self.g_pool.get_timestamp(), "record": True, }) self.counter = 0 else: self.counter -= 1 ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) if self.counter <= 0: # last sample before counter done and moving on audio.tink() logger.debug( "Sampled {} datapoints. Stopping to sample. Looking for steady marker again." .format(self.counter_max)) self.notify_all({ "subject": "calibration.marker_sample_completed", "timestamp": self.g_pool.get_timestamp(), "record": True, }) # Always save pupil positions self.pupil_list.extend(events["pupil"]) if self.counter: if len(self.markers): self.button.status_text = "Sampling Gaze Data" else: self.button.status_text = "Marker Lost" else: self.button.status_text = "Looking for Marker" # Stop if autostop condition is satisfied: if self.auto_stop >= self.auto_stop_max: self.auto_stop = 0 self.stop() else: pass def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: draw_points_norm([self.smooth_pos], size=15, color=RGBA(1.0, 1.0, 0.0, 0.5)) if self.active and len(self.markers): # draw the largest ellipse of all detected markers for marker in self.markers: e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=GL_POLYGON) # draw indicator on the first detected marker if self.counter and self.markers[0]["marker_type"] == "Ref": e = self.markers[0]["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.counter_max, ) indicator = [e[0]] + pts[self.counter:].tolist()[::-1] + [e[0]] draw_polyline(indicator, color=RGBA(0.1, 0.5, 0.7, 0.8), line_type=GL_POLYGON) # draw indicator on the stop marker(s) if self.auto_stop: for marker in self.markers: if marker["marker_type"] == "Stop": e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.auto_stop_max, ) indicator = [e[0]] + pts[self.auto_stop:].tolist() + [ e[0] ] draw_polyline( indicator, color=RGBA(8.0, 0.1, 0.1, 0.8), line_type=GL_POLYGON, ) else: pass def deinit_ui(self): """gets called when the plugin get terminated. This happens either voluntarily or forced. if you have an atb bar or glfw window destroy it here. """ if self.active: self.stop() super().deinit_ui()
class ScreenMarkerChoreographyPlugin( MonitorSelectionMixin, CalibrationChoreographyPlugin ): """Calibrate using a marker on your screen We use a ring detector that moves across the screen to 9 sites Points are collected at sites - not between """ label = "Screen Marker Calibration" @classmethod def selection_label(cls) -> str: return "Screen Marker" @classmethod def selection_order(cls) -> float: return 1.0 @staticmethod def get_list_of_markers_to_show(mode: ChoreographyMode) -> list: if ChoreographyMode.CALIBRATION == mode: return [(0.5, 0.5), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0), (0.0, 0.0)] if ChoreographyMode.VALIDATION == mode: return [(0.5, 1.0), (1.0, 0.5), (0.5, 0.0), (0.0, 0.5)] raise ValueError(f"Unknown mode {mode}") def __init__( self, g_pool, fullscreen=True, marker_scale=1.0, sample_duration=40, monitor_name=None, **kwargs, ): super().__init__(g_pool, **kwargs) # Public properties self.selected_monitor_name = monitor_name self.is_fullscreen = fullscreen self.sample_duration = sample_duration # Private properties self.__current_list_of_markers_to_show = [] self.__currently_shown_marker_position = None self.__ref_count_for_current_marker_position = 0 self.__previously_detected_markers = [] self.__circle_tracker = CircleTracker() self.__marker_window = MarkerWindowController(marker_scale=marker_scale) self.__marker_window.add_observer( "on_window_did_close", self._on_window_did_close ) def get_init_dict(self): d = {} d["fullscreen"] = self.is_fullscreen d["marker_scale"] = self.__marker_window.marker_scale d["monitor_name"] = self.selected_monitor_name return d ### Public - Plugin @classmethod def _choreography_description_text(cls) -> str: return "Calibrate gaze parameters using a screen based animation." def _init_custom_menu_ui_elements(self) -> list: self.__ui_selector_monitor_name = ui.Selector( "selected_monitor_name", self, label="Monitor", labels=self.currently_connected_monitor_names(), selection=self.currently_connected_monitor_names(), ) self.__ui_switch_is_fullscreen = ui.Switch( "is_fullscreen", self, label="Use fullscreen" ) self.__ui_slider_marker_scale = ui.Slider( "marker_scale", self.__marker_window, label="Marker size", min=0.5, max=2.0, step=0.1, ) self.__ui_slider_sample_duration = ui.Slider( "sample_duration", self, label="Sample duration", min=10, max=100, step=1 ) return [ self.__ui_selector_monitor_name, self.__ui_switch_is_fullscreen, self.__ui_slider_marker_scale, self.__ui_slider_sample_duration, ] def deinit_ui(self): self.__marker_window.close_window() super().deinit_ui() def recent_events(self, events): super().recent_events(events) frame = events.get("frame") state = self.__marker_window.window_state should_animate = True if not frame: return self.__marker_window.update_state() if isinstance(state, MarkerWindowStateClosed): return elif isinstance(state, MarkerWindowStateOpened): assert self.is_active # Sanity check pass # Continue with processing the frame else: raise UnhandledMarkerWindowStateError(state) # Always save pupil positions self.pupil_list.extend(events["pupil"]) # Detect reference circle marker detected_marker = self.__detect_reference_circle_marker(frame.gray) # Signal marker window controller that a marker was detected (for feedback) self.__marker_window.is_marker_detected = detected_marker is not None if isinstance(state, MarkerWindowStateIdle): assert self.__currently_shown_marker_position is None # Sanity check if self.__current_list_of_markers_to_show: self.__currently_shown_marker_position = ( self.__current_list_of_markers_to_show.pop(0) ) logger.debug( f"Moving screen marker to site at {self.__currently_shown_marker_position}" ) self.__marker_window.show_marker( marker_position=self.__currently_shown_marker_position, should_animate=should_animate, ) return else: # No more markers to show; stop calibration choreography. self._signal_should_stop(mode=self.current_mode) return if isinstance(state, MarkerWindowStateAnimatingInMarker): assert self.__currently_shown_marker_position is not None # Sanity check pass # No-op elif isinstance(state, MarkerWindowStateShowingMarker): assert self.__currently_shown_marker_position is not None # Sanity check if detected_marker is not None: ref = {} ref["norm_pos"] = detected_marker["norm_pos"] ref["screen_pos"] = detected_marker["img_pos"] ref["timestamp"] = frame.timestamp self.ref_list.append(ref) should_move_to_next_marker = len(self.ref_list) == self.sample_duration * ( self.__ref_count_for_current_marker_position + 1 ) if should_move_to_next_marker: # Finished collecting samples for current active site self.__currently_shown_marker_position = None self.__ref_count_for_current_marker_position += 1 self.__marker_window.hide_marker(should_animate=should_animate) elif isinstance(state, MarkerWindowStateAnimatingOutMarker): assert self.__currently_shown_marker_position is None # Sanity check pass # No-op else: raise UnhandledMarkerWindowStateError(state) # Update UI self.__marker_window.draw_window() self.status_text = self.__currently_shown_marker_position def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if not self.is_active: return markers = self.__previously_detected_markers for marker in markers: e = marker["ellipses"][-1] # outermost ellipse pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0)) if len(markers) > 1: draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON) ### Internal def _perform_start(self): if not self.g_pool.capture.online: logger.error( f"{self.current_mode.label} requiers world capture video input." ) return self.__current_list_of_markers_to_show = self.get_list_of_markers_to_show( mode=self.current_mode, ) self.__currently_shown_marker_position = None self.__ref_count_for_current_marker_position = 0 super()._perform_start() self.__marker_window.open_window( title=self.current_mode.label, monitor_name=self.selected_monitor_name, is_fullscreen=self.is_fullscreen, ) def _perform_stop(self): self.__marker_window.close_window() super()._perform_stop() ### Private def _on_window_did_close(self): self._signal_should_stop(mode=self.current_mode) def __detect_reference_circle_marker(self, gray_img): # Detect all circular markers circle_markers = self.__circle_tracker.update(gray_img) # Only keep Ref markers circle_markers = [ marker for marker in circle_markers if marker["marker_type"] == "Ref" ] # Store detected Ref markers for debugging/visualization self.__previously_detected_markers = circle_markers if len(circle_markers) == 0: return None elif len(circle_markers) == 1: return circle_markers[0] else: logger.warning( f"{len(circle_markers)} markers detected. Please remove all the other markers" ) return circle_markers[0]
class Screen_Marker_Calibration(Calibration_Plugin): """Calibrate using a marker on your screen We use a ring detector that moves across the screen to 9 sites Points are collected at sites - not between """ def __init__( self, g_pool, fullscreen=True, marker_scale=1.0, sample_duration=40, monitor_idx=0, ): super().__init__(g_pool) self.screen_marker_state = 0.0 self.sample_duration = sample_duration # number of frames to sample per site self.lead_in = 25 # frames of marker shown before starting to sample self.lead_out = 5 # frames of markers shown after sampling is donw self.active_site = None self.sites = [] self.display_pos = -1.0, -1.0 self.on_position = False self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.monitor_idx = monitor_idx self.fullscreen = fullscreen self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font("opensans", get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0)) self.glfont.set_align_string(v_align="center") # UI Platform tweaks if system() == "Linux": self.window_position_default = (0, 0) elif system() == "Windows": self.window_position_default = (8, 90) else: self.window_position_default = (0, 0) self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.menu.label = "Screen Marker Calibration" def get_monitors_idx_list(): monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()] return range(len(monitors)), monitors if self.monitor_idx not in get_monitors_idx_list()[0]: logger.warning( "Monitor at index %s no longer availalbe using default" % self.monitor_idx) self.monitor_idx = 0 self.menu.append( ui.Info_Text( "Calibrate gaze parameters using a screen based animation.")) self.menu.append( ui.Selector( "monitor_idx", self, selection_getter=get_monitors_idx_list, label="Monitor", )) self.menu.append(ui.Switch("fullscreen", self, label="Use fullscreen")) self.menu.append( ui.Slider("marker_scale", self, step=0.1, min=0.5, max=2.0, label="Marker size")) self.menu.append( ui.Slider( "sample_duration", self, step=1, min=10, max=100, label="Sample duration", )) def start(self): if not self.g_pool.capture.online: logger.error("{} requiers world capture video input.".format( self.mode_pretty)) return super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) if self.g_pool.detection_mapping_mode == "3d": if self.mode == "calibration": self.sites = [ (0.5, 0.5), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0), (0.0, 0.0), ] else: self.sites = [(0.25, 0.5), (0.5, 0.25), (0.75, 0.5), (0.5, 0.75)] else: if self.mode == "calibration": self.sites = [ (0.25, 0.5), (0, 0.5), (0.0, 1.0), (0.5, 1.0), (1.0, 1.0), (1.0, 0.5), (1.0, 0.0), (0.5, 0.0), (0.0, 0.0), (0.75, 0.5), ] else: self.sites = [ (0.5, 0.5), (0.25, 0.25), (0.25, 0.75), (0.75, 0.75), (0.75, 0.25), ] self.active_site = self.sites.pop(0) self.active = True self.ref_list = [] self.pupil_list = [] self.clicks_to_close = 5 self.open_window(self.mode_pretty) def open_window(self, title="new_window"): if not self._window: if self.fullscreen: try: monitor = glfwGetMonitors()[self.monitor_idx] except: logger.warning( "Monitor at index %s no longer availalbe using default" % self.monitor_idx) self.monitor_idx = 0 monitor = glfwGetMonitors()[self.monitor_idx] width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode( monitor) else: monitor = None width, height = 640, 360 self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext()) if not self.fullscreen: glfwSetWindowPos( self._window, self.window_position_default[0], self.window_position_default[1], ) glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN) # Register callbacks glfwSetFramebufferSizeCallback(self._window, on_resize) glfwSetKeyCallback(self._window, self.on_window_key) glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button) on_resize(self._window, *glfwGetFramebufferSize(self._window)) # gl_state settings active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) def on_window_key(self, window, key, scancode, action, mods): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: self.clicks_to_close = 0 def on_window_mouse_button(self, window, button, action, mods): if action == GLFW_PRESS: self.clicks_to_close -= 1 def stop(self): # TODO: redundancy between all gaze mappers -> might be moved to parent class audio.say("Stopping {}".format(self.mode_pretty)) logger.info("Stopping {}".format(self.mode_pretty)) self.smooth_pos = 0, 0 self.counter = 0 self.close_window() self.active = False self.button.status_text = "" if self.mode == "calibration": finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == "accuracy_test": self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def close_window(self): if self._window: # enable mouse display active_window = glfwGetCurrentContext() glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL) glfwDestroyWindow(self._window) self._window = None glfwMakeContextCurrent(active_window) def recent_events(self, events): frame = events.get("frame") if self.active and frame: gray_img = frame.gray if self.clicks_to_close <= 0: self.stop() return # Update the marker self.markers = self.circle_tracker.update(gray_img) # Screen marker takes only Ref marker self.markers = [ marker for marker in self.markers if marker["marker_type"] == "Ref" ] if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]["img_pos"] self.pos = self.markers[0]["norm_pos"] else: self.pos = None # indicate that no reference is detected # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers". format(len(self.markers))) # only save a valid ref position if within sample window of calibration routine on_position = (self.lead_in < self.screen_marker_state < (self.lead_in + self.sample_duration)) if on_position and len(self.markers): ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # Always save pupil positions self.pupil_list.extend(events["pupil"]) if on_position and len(self.markers) and events.get( "fixations", []): fixation_boost = 5 self.screen_marker_state = min( self.sample_duration + self.lead_in, self.screen_marker_state + fixation_boost, ) # Animate the screen marker if (self.screen_marker_state < self.sample_duration + self.lead_in + self.lead_out): if len(self.markers) or not on_position: self.screen_marker_state += 1 else: self.screen_marker_state = 0 if not self.sites: self.stop() return self.active_site = self.sites.pop(0) logger.debug("Moving screen marker to site at {} {}".format( *self.active_site)) # use np.arrays for per element wise math self.display_pos = np.array(self.active_site) self.on_position = on_position self.button.status_text = "{}".format(self.active_site) if self._window: self.gl_display_in_window() def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active: for marker in self.markers: e = marker["ellipses"][-1] # outermost ellipse pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON) def gl_display_in_window(self): active_window = glfwGetCurrentContext() if glfwWindowShouldClose(self._window): self.close_window() return glfwMakeContextCurrent(self._window) clear_gl_screen() hdpi_factor = getHDPIFactor(self._window) r = self.marker_scale * hdpi_factor gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() p_window_size = glfwGetFramebufferSize(self._window) gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1) # Switch back to Model View Matrix gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() def map_value(value, in_range=(0, 1), out_range=(0, 1)): ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0]) return (value - in_range[0]) * ratio + out_range[0] pad = 90 * r screen_pos = ( map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)), map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)), ) alpha = interp_fn( self.screen_marker_state, 0.0, 1.0, float(self.sample_duration + self.lead_in + self.lead_out), float(self.lead_in), float(self.sample_duration + self.lead_in), ) r2 = 2 * r draw_points([screen_pos], size=60 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.9) draw_points([screen_pos], size=38 * r2, color=RGBA(1.0, 1.0, 1.0, alpha), sharpness=0.8) draw_points([screen_pos], size=19 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.55) # some feedback on the detection state color = (RGBA(0.0, 0.8, 0.0, alpha) if len(self.markers) and self.on_position else RGBA(0.8, 0.0, 0.0, alpha)) draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5) if self.clicks_to_close < 5: self.glfont.set_size(int(p_window_size[0] / 30.0)) self.glfont.draw_text( p_window_size[0] / 2.0, p_window_size[1] / 4.0, "Touch {} more times to cancel {}.".format( self.clicks_to_close, self.mode_pretty), ) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) def get_init_dict(self): d = {} d["fullscreen"] = self.fullscreen d["marker_scale"] = self.marker_scale d["monitor_idx"] = self.monitor_idx return d def deinit_ui(self): """gets called when the plugin get terminated. either voluntarily or forced. """ if self.active: self.stop() if self._window: self.close_window() super().deinit_ui()
class Manual_Marker_Calibration(Calibration_Plugin): """ CircleTracker looks for proper markers Using at least 9 positions/points within the FOV Ref detector will direct one to good positions with audio cues Calibration only collects data at the good positions """ def __init__(self, g_pool): super().__init__(g_pool) self.pos = None self.smooth_pos = 0., 0. self.smooth_vel = 0. self.sample_site = (-2, -2) self.counter = 0 self.counter_max = 30 self.stop_marker_found = False self.auto_stop = 0 self.auto_stop_max = 30 self.menu = None self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.menu.label = "Manual Calibration" self.menu.append( ui.Info_Text("Calibrate gaze parameters using a handheld marker.")) def start(self): super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) self.active = True self.ref_list = [] self.pupil_list = [] def stop(self): audio.say("Stopping {}".format(self.mode_pretty)) logger.info('Stopping {}'.format(self.mode_pretty)) self.screen_marker_state = 0 self.active = False # self.close_window() self.button.status_text = '' if self.mode == 'calibration': finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == 'accuracy_test': self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def on_notify(self, notification): ''' Reacts to notifications: ``calibration.should_start``: Starts the calibration procedure ``calibration.should_stop``: Stops the calibration procedure Emits notifications: ``calibration.started``: Calibration procedure started ``calibration.stopped``: Calibration procedure stopped ``calibration.marker_found``: Steady marker found ``calibration.marker_moved_too_quickly``: Marker moved too quickly ``calibration.marker_sample_completed``: Enough data points sampled ''' super().on_notify(notification) def recent_events(self, events): """ gets called once every frame. reference positon need to be published to shared_pos if no reference was found, publish 0,0 """ frame = events.get('frame') if self.active and frame: recent_pupil_positions = events['pupil_positions'] gray_img = frame.gray # Update the marker self.markers = self.circle_tracker.update(gray_img) self.stop_marker_found = False if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]['img_pos'] self.pos = self.markers[0]['norm_pos'] # Check if there are stop markers for marker in self.markers: if marker['marker_type'] == 'Stop': self.auto_stop += 1 self.stop_marker_found = True break else: self.pos = None # indicate that no reference is detected if self.stop_marker_found is False: self.auto_stop = 0 # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers". format(len(self.markers))) # tracking logic if len(self.markers) and not self.stop_marker_found: # start counter if ref is resting in place and not at last sample site # calculate smoothed manhattan velocity smoother = 0.3 smooth_pos = np.array(self.smooth_pos) pos = np.array(self.pos) new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos) smooth_vel_vec = new_smooth_pos - smooth_pos smooth_pos = new_smooth_pos self.smooth_pos = list(smooth_pos) #manhattan distance for velocity new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1]) self.smooth_vel = self.smooth_vel + smoother * ( new_vel - self.smooth_vel) #distance to last sampled site sample_ref_dist = smooth_pos - np.array(self.sample_site) sample_ref_dist = abs(sample_ref_dist[0]) + abs( sample_ref_dist[1]) # start counter if ref is resting in place and not at last sample site if self.counter <= 0: if self.smooth_vel < 0.01 and sample_ref_dist > 0.1: self.sample_site = self.smooth_pos audio.beep() logger.debug( "Steady marker found. Starting to sample {} datapoints" .format(self.counter_max)) self.notify_all({ 'subject': 'calibration.marker_found', 'timestamp': self.g_pool.get_timestamp(), 'record': True }) self.counter = self.counter_max if self.counter > 0: if self.smooth_vel > 0.01: audio.tink() logger.warning( "Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again." .format(self.counter_max - self.counter)) self.notify_all({ 'subject': 'calibration.marker_moved_too_quickly', 'timestamp': self.g_pool.get_timestamp(), 'record': True }) self.counter = 0 else: self.counter -= 1 ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) if events.get('fixations', []): self.counter -= 5 if self.counter <= 0: #last sample before counter done and moving on audio.tink() logger.debug( "Sampled {} datapoints. Stopping to sample. Looking for steady marker again." .format(self.counter_max)) self.notify_all({ 'subject': 'calibration.marker_sample_completed', 'timestamp': self.g_pool.get_timestamp(), 'record': True }) # Always save pupil positions for p_pt in recent_pupil_positions: if p_pt['confidence'] > self.pupil_confidence_threshold: self.pupil_list.append(p_pt) if self.counter: if len(self.markers): self.button.status_text = 'Sampling Gaze Data' else: self.button.status_text = 'Marker Lost' else: self.button.status_text = 'Looking for Marker' # Stop if autostop condition is satisfied: if self.auto_stop >= self.auto_stop_max: self.auto_stop = 0 self.stop() else: pass def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: draw_points_norm([self.smooth_pos], size=15, color=RGBA(1., 1., 0., .5)) if self.active and len(self.markers): # draw the largest ellipse of all detected markers for marker in self.markers: e = marker['ellipses'][-1] pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15) draw_polyline(pts, color=RGBA(0., 1., 0, 1.)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1., 0., 0., .5), line_type=GL_POLYGON) # draw indicator on the first detected marker if self.counter and self.markers[0]['marker_type'] == 'Ref': e = self.markers[0]['ellipses'][-1] pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.counter_max) indicator = [e[0]] + pts[self.counter:].tolist()[::-1] + [e[0]] draw_polyline(indicator, color=RGBA(0.1, .5, .7, .8), line_type=GL_POLYGON) # draw indicator on the stop marker(s) if self.auto_stop: for marker in self.markers: if marker['marker_type'] == 'Stop': e = marker['ellipses'][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.auto_stop_max) indicator = [e[0]] + pts[self.auto_stop:].tolist() + [ e[0] ] draw_polyline(indicator, color=RGBA(8., 0.1, 0.1, .8), line_type=GL_POLYGON) else: pass def deinit_ui(self): """gets called when the plugin get terminated. This happens either voluntarily or forced. if you have an atb bar or glfw window destroy it here. """ if self.active: self.stop() super().deinit_ui()
class Single_Marker_Calibration(Calibration_Plugin): """Calibrate using a single marker. Move your head for example in a spiral motion while gazing at the marker to quickly sample a wide range gaze angles. """ def __init__(self, g_pool, fullscreen=True, marker_scale=1.0, sample_duration=40): super().__init__(g_pool) self.screen_marker_state = 0. self.lead_in = 25 # frames of marker shown before starting to sample self.display_pos = (.5, .5) self.on_position = False self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.fullscreen = fullscreen self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font('opensans', get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0)) self.glfont.set_align_string(v_align='center') # UI Platform tweaks if system() == 'Linux': self.window_position_default = (0, 0) elif system() == 'Windows': self.window_position_default = (8, 31) else: self.window_position_default = (0, 0) self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.monitor_idx = 0 self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()] #primary_monitor = glfwGetPrimaryMonitor() self.menu.append( ui.Info_Text( "Calibrate gaze parameters using a single gae targets and active head movements." )) self.menu.append( ui.Selector('monitor_idx', self, selection=range(len(self.monitor_names)), labels=self.monitor_names, label='Monitor')) self.menu.append(ui.Switch('fullscreen', self, label='Use fullscreen')) self.menu.append( ui.Slider('marker_scale', self, step=0.1, min=0.5, max=2.0, label='Marker size')) def start(self): if not self.g_pool.capture.online: logger.error("Calibration required world capture video input.") return super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) self.active = True self.ref_list = [] self.pupil_list = [] self.clicks_to_close = 5 self.open_window(self.mode_pretty) def open_window(self, title='new_window'): if not self._window: if self.fullscreen: monitor = glfwGetMonitors()[self.monitor_idx] width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode( monitor) else: monitor = None width, height = 640, 360 self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext()) if not self.fullscreen: glfwSetWindowPos(self._window, self.window_position_default[0], self.window_position_default[1]) glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN) # Register callbacks glfwSetFramebufferSizeCallback(self._window, on_resize) glfwSetKeyCallback(self._window, self.on_window_key) glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button) on_resize(self._window, *glfwGetFramebufferSize(self._window)) # gl_state settings active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) def on_window_key(self, window, key, scancode, action, mods): if action == GLFW_PRESS: if self.mode == 'calibration': target_key = GLFW_KEY_C else: target_key = GLFW_KEY_T if key == GLFW_KEY_ESCAPE or key == target_key: self.clicks_to_close = 0 def on_window_mouse_button(self, window, button, action, mods): if action == GLFW_PRESS: self.clicks_to_close -= 1 def stop(self): # TODO: redundancy between all gaze mappers -> might be moved to parent class audio.say("Stopping {}".format(self.mode_pretty)) logger.info('Stopping {}'.format(self.mode_pretty)) self.smooth_pos = 0, 0 self.counter = 0 self.close_window() self.active = False self.button.status_text = '' if self.mode == 'calibration': finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == 'accuracy_test': self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def close_window(self): if self._window: # enable mouse display active_window = glfwGetCurrentContext() glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL) glfwDestroyWindow(self._window) self._window = None glfwMakeContextCurrent(active_window) def recent_events(self, events): frame = events.get('frame') if self.active and frame: recent_pupil_positions = events['pupil_positions'] gray_img = frame.gray if self.clicks_to_close <= 0: self.stop() return # Update the marker self.markers = self.circle_tracker.update(gray_img) # Screen marker takes only Ref marker self.markers = [ marker for marker in self.markers if marker['marker_type'] == 'Ref' ] if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]['img_pos'] self.pos = self.markers[0]['norm_pos'] else: self.pos = None # indicate that no reference is detected # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers". format(len(self.markers))) # only save a valid ref position if within sample window of calibraiton routine on_position = self.lead_in < self.screen_marker_state if on_position and len(self.markers): ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # always save pupil positions for p_pt in recent_pupil_positions: if p_pt['confidence'] > self.pupil_confidence_threshold: self.pupil_list.append(p_pt) # Animate the screen marker if len(self.markers) or not on_position: self.screen_marker_state += 1 # use np.arrays for per element wise math self.on_position = on_position if self._window: self.gl_display_in_window() def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active: for marker in self.markers: e = marker['ellipses'][-1] # outermost ellipse pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15) draw_polyline(pts, 1, RGBA(0., 1., 0., 1.)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1., 0., 0., .5), line_type=gl.GL_POLYGON) def gl_display_in_window(self): active_window = glfwGetCurrentContext() if glfwWindowShouldClose(self._window): self.close_window() return glfwMakeContextCurrent(self._window) clear_gl_screen() hdpi_factor = glfwGetFramebufferSize( self._window)[0] / glfwGetWindowSize(self._window)[0] r = self.marker_scale * hdpi_factor gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() p_window_size = glfwGetFramebufferSize(self._window) gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1) # Switch back to Model View Matrix gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() def map_value(value, in_range=(0, 1), out_range=(0, 1)): ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0]) return (value - in_range[0]) * ratio + out_range[0] pad = 90 * r screen_pos = map_value( self.display_pos[0], out_range=(pad, p_window_size[0] - pad)), map_value( self.display_pos[1], out_range=(p_window_size[1] - pad, pad)) alpha = 1.0 #interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in)) r2 = 2 * r draw_points([screen_pos], size=60 * r2, color=RGBA(0., 0., 0., alpha), sharpness=0.9) draw_points([screen_pos], size=38 * r2, color=RGBA(1., 1., 1., alpha), sharpness=0.8) draw_points([screen_pos], size=19 * r2, color=RGBA(0., 0., 0., alpha), sharpness=0.55) # some feedback on the detection state color = RGBA(0., .8, 0., alpha) if len( self.markers) and self.on_position else RGBA(0.8, 0., 0., alpha) draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5) if self.clicks_to_close < 5: self.glfont.set_size(int(p_window_size[0] / 30.)) self.glfont.draw_text( p_window_size[0] / 2., p_window_size[1] / 4., 'Touch {} more times to cancel calibration.'.format( self.clicks_to_close)) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) def get_init_dict(self): d = {} d['fullscreen'] = self.fullscreen d['marker_scale'] = self.marker_scale return d def deinit_ui(self): """gets called when the plugin get terminated. either voluntarily or forced. """ if self.active: self.stop() if self._window: self.close_window() super().deinit_ui()
def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20): # ipc setup import zmq import zmq_tools zmq_ctx = zmq.Context() process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from time import sleep from video_capture import File_Source, EndofVideoFileError from circle_detector import CircleTracker try: src = File_Source(Empty(), source_path, timed_playback=False) frame = src.get_frame() logger.info('Starting calibration marker detection...') frame_count = src.get_frame_count() queue = [] circle_tracker = CircleTracker() while True: while process_pipe.new_data: topic, n = process_pipe.recv() if topic == 'terminate': process_pipe.send(topic='exception', payload={"reason": "User terminated."}) logger.debug("Process terminated") sleep(1.0) return progress = 100. * frame.index / frame_count markers = [ m for m in circle_tracker.update(frame.gray) if m['marker_type'] == 'Ref' ] if len(markers): ref = { "norm_pos": markers[0]['norm_pos'], "screen_pos": markers[0]['img_pos'], "timestamp": frame.timestamp, 'index': frame.index } queue.append((progress, ref)) else: queue.append((progress, None)) if len(queue) > batch_size: # dequeue batch data = queue[:batch_size] del queue[:batch_size] process_pipe.send(topic='progress', payload={'data': data}) frame = src.get_frame() except EndofVideoFileError: process_pipe.send(topic='progress', payload={'data': queue}) process_pipe.send(topic='finished', payload={}) logger.debug("Process finished") except Exception: import traceback process_pipe.send(topic='exception', payload={'reason': traceback.format_exc()}) logger.debug("Process raised Exception") sleep(1.0)
def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20): # ipc setup import zmq import zmq_tools zmq_ctx = zmq.Context() process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from time import sleep from video_capture import init_playback_source, EndofVideoError from circle_detector import CircleTracker try: src = init_playback_source(Empty(), source_path, timing=None) frame = src.get_frame() logger.info("Starting calibration marker detection...") frame_count = src.get_frame_count() queue = [] circle_tracker = CircleTracker() while True: while process_pipe.new_data: topic, n = process_pipe.recv() if topic == "terminate": process_pipe.send({ "topic": "exception", "reason": "User terminated." }) logger.debug("Process terminated") sleep(1.0) return progress = 100.0 * frame.index / frame_count markers = [ m for m in circle_tracker.update(frame.gray) if m["marker_type"] == "Ref" ] if len(markers): ref = { "norm_pos": markers[0]["norm_pos"], "screen_pos": markers[0]["img_pos"], "timestamp": frame.timestamp, "index_range": tuple(range(frame.index - 5, frame.index + 5)), "index": frame.index, } queue.append((progress, ref)) else: queue.append((progress, None)) if len(queue) > batch_size: # dequeue batch data = queue[:batch_size] del queue[:batch_size] process_pipe.send({"topic": "progress", "data": data}) frame = src.get_frame() except EndofVideoError: process_pipe.send({"topic": "progress", "data": queue}) process_pipe.send({"topic": "finished"}) logger.debug("Process finished") except Exception: import traceback process_pipe.send({ "topic": "exception", "reason": traceback.format_exc() }) logger.debug("Process raised Exception") sleep(1.0)
class Screen_Marker_Calibration(Calibration_Plugin): """Calibrate using a marker on your screen We use a ring detector that moves across the screen to 9 sites Points are collected at sites - not between """ def __init__( self, g_pool, fullscreen=True, marker_scale=1.0, sample_duration=40, monitor_idx=0, ): super().__init__(g_pool) self.screen_marker_state = 0.0 self.sample_duration = sample_duration # number of frames to sample per site self.lead_in = 25 # frames of marker shown before starting to sample self.lead_out = 5 # frames of markers shown after sampling is donw self.active_site = None self.sites = [] self.display_pos = -1.0, -1.0 self.on_position = False self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.monitor_idx = monitor_idx self.fullscreen = fullscreen self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font("opensans", get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0)) self.glfont.set_align_string(v_align="center") # UI Platform tweaks if system() == "Linux": self.window_position_default = (0, 0) elif system() == "Windows": self.window_position_default = (8, 90) else: self.window_position_default = (0, 0) self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.menu.label = "Screen Marker Calibration" def get_monitors_idx_list(): monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()] return range(len(monitors)), monitors if self.monitor_idx not in get_monitors_idx_list()[0]: logger.warning( "Monitor at index %s no longer availalbe using default" % self.monitor_idx ) self.monitor_idx = 0 self.menu.append( ui.Info_Text("Calibrate gaze parameters using a screen based animation.") ) self.menu.append( ui.Selector( "monitor_idx", self, selection_getter=get_monitors_idx_list, label="Monitor", ) ) self.menu.append(ui.Switch("fullscreen", self, label="Use fullscreen")) self.menu.append( ui.Slider( "marker_scale", self, step=0.1, min=0.5, max=2.0, label="Marker size" ) ) self.menu.append( ui.Slider( "sample_duration", self, step=1, min=10, max=100, label="Sample duration", ) ) def start(self): if not self.g_pool.capture.online: logger.error( "{} requiers world capture video input.".format(self.mode_pretty) ) return super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) if self.g_pool.detection_mapping_mode == "3d": if self.mode == "calibration": self.sites = [ (0.5, 0.5), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0), (0.0, 0.0), ] else: self.sites = [(0.25, 0.5), (0.5, 0.25), (0.75, 0.5), (0.5, 0.75)] else: if self.mode == "calibration": self.sites = [ (0.25, 0.5), (0, 0.5), (0.0, 1.0), (0.5, 1.0), (1.0, 1.0), (1.0, 0.5), (1.0, 0.0), (0.5, 0.0), (0.0, 0.0), (0.75, 0.5), ] else: self.sites = [ (0.5, 0.5), (0.25, 0.25), (0.25, 0.75), (0.75, 0.75), (0.75, 0.25), ] self.active_site = self.sites.pop(0) self.active = True self.ref_list = [] self.pupil_list = [] self.clicks_to_close = 5 self.open_window(self.mode_pretty) def open_window(self, title="new_window"): if not self._window: if self.fullscreen: try: monitor = glfwGetMonitors()[self.monitor_idx] except: logger.warning( "Monitor at index %s no longer availalbe using default" % self.monitor_idx ) self.monitor_idx = 0 monitor = glfwGetMonitors()[self.monitor_idx] width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode( monitor ) else: monitor = None width, height = 640, 360 self._window = glfwCreateWindow( width, height, title, monitor=monitor, share=glfwGetCurrentContext() ) if not self.fullscreen: glfwSetWindowPos( self._window, self.window_position_default[0], self.window_position_default[1], ) glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN) # Register callbacks glfwSetFramebufferSizeCallback(self._window, on_resize) glfwSetKeyCallback(self._window, self.on_window_key) glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button) on_resize(self._window, *glfwGetFramebufferSize(self._window)) # gl_state settings active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) def on_window_key(self, window, key, scancode, action, mods): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: self.clicks_to_close = 0 def on_window_mouse_button(self, window, button, action, mods): if action == GLFW_PRESS: self.clicks_to_close -= 1 def stop(self): # TODO: redundancy between all gaze mappers -> might be moved to parent class audio.say("Stopping {}".format(self.mode_pretty)) logger.info("Stopping {}".format(self.mode_pretty)) self.smooth_pos = 0, 0 self.counter = 0 self.close_window() self.active = False self.button.status_text = "" if self.mode == "calibration": finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == "accuracy_test": self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def close_window(self): if self._window: # enable mouse display active_window = glfwGetCurrentContext() glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL) glfwDestroyWindow(self._window) self._window = None glfwMakeContextCurrent(active_window) def recent_events(self, events): frame = events.get("frame") if self.active and frame: gray_img = frame.gray if self.clicks_to_close <= 0: self.stop() return # Update the marker self.markers = self.circle_tracker.update(gray_img) # Screen marker takes only Ref marker self.markers = [ marker for marker in self.markers if marker["marker_type"] == "Ref" ] if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]["img_pos"] self.pos = self.markers[0]["norm_pos"] else: self.pos = None # indicate that no reference is detected # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers".format( len(self.markers) ) ) # only save a valid ref position if within sample window of calibration routine on_position = ( self.lead_in < self.screen_marker_state < (self.lead_in + self.sample_duration) ) if on_position and len(self.markers): ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # Always save pupil positions self.pupil_list.extend(events["pupil"]) if on_position and len(self.markers) and events.get("fixations", []): fixation_boost = 5 self.screen_marker_state = min( self.sample_duration + self.lead_in, self.screen_marker_state + fixation_boost, ) # Animate the screen marker if ( self.screen_marker_state < self.sample_duration + self.lead_in + self.lead_out ): if len(self.markers) or not on_position: self.screen_marker_state += 1 else: self.screen_marker_state = 0 if not self.sites: self.stop() return self.active_site = self.sites.pop(0) logger.debug( "Moving screen marker to site at {} {}".format(*self.active_site) ) # use np.arrays for per element wise math self.display_pos = np.array(self.active_site) self.on_position = on_position self.button.status_text = "{}".format(self.active_site) if self._window: self.gl_display_in_window() def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active: for marker in self.markers: e = marker["ellipses"][-1] # outermost ellipse pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0)) if len(self.markers) > 1: draw_polyline( pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON ) def gl_display_in_window(self): active_window = glfwGetCurrentContext() if glfwWindowShouldClose(self._window): self.close_window() return glfwMakeContextCurrent(self._window) clear_gl_screen() hdpi_factor = getHDPIFactor(self._window) r = self.marker_scale * hdpi_factor gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() p_window_size = glfwGetFramebufferSize(self._window) gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1) # Switch back to Model View Matrix gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() def map_value(value, in_range=(0, 1), out_range=(0, 1)): ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0]) return (value - in_range[0]) * ratio + out_range[0] pad = 90 * r screen_pos = ( map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)), map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)), ) alpha = interp_fn( self.screen_marker_state, 0.0, 1.0, float(self.sample_duration + self.lead_in + self.lead_out), float(self.lead_in), float(self.sample_duration + self.lead_in), ) r2 = 2 * r draw_points( [screen_pos], size=60 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.9 ) draw_points( [screen_pos], size=38 * r2, color=RGBA(1.0, 1.0, 1.0, alpha), sharpness=0.8 ) draw_points( [screen_pos], size=19 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.55 ) # some feedback on the detection state color = ( RGBA(0.0, 0.8, 0.0, alpha) if len(self.markers) and self.on_position else RGBA(0.8, 0.0, 0.0, alpha) ) draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5) if self.clicks_to_close < 5: self.glfont.set_size(int(p_window_size[0] / 30.0)) self.glfont.draw_text( p_window_size[0] / 2.0, p_window_size[1] / 4.0, "Touch {} more times to cancel {}.".format( self.clicks_to_close, self.mode_pretty ), ) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) def get_init_dict(self): d = {} d["fullscreen"] = self.fullscreen d["marker_scale"] = self.marker_scale d["monitor_idx"] = self.monitor_idx return d def deinit_ui(self): """gets called when the plugin get terminated. either voluntarily or forced. """ if self.active: self.stop() if self._window: self.close_window() super().deinit_ui()
class Participant_Driven_Screen_Marker_Calibration(Calibration_Plugin): """ Calibrate using on screen markers. We use a ring detector that moves across the screen to 9 sites Points are collected at sites - not between Points are collected after space key is pressed """ def __init__( self, g_pool, fullscreen=True, marker_scale=1.0, sample_duration=40, monitor_idx=1 ): super().__init__(g_pool) self.detected = False self.space_key_was_pressed = False self.screen_marker_state = 0. self.sample_duration = sample_duration # number of frames to sample per site self.fixation_boost = sample_duration/2. self.lead_in = 25 #frames of marker shown before starting to sample self.lead_out = 5 #frames of markers shown after sampling is donw self.monitor_idx = monitor_idx self.active_site = None self.sites = [] self.display_pos = -1., -1. self.on_position = False self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.button = None self.fullscreen = fullscreen self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font('opensans',get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2,0.5,0.9,1.0)) self.glfont.set_align_string(v_align='center') # UI Platform tweaks if system() == 'Linux': self.window_position_default = (0, 0) elif system() == 'Windows': self.window_position_default = (8, 31) else: self.window_position_default = (0, 0) self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.menu.label = "Participant Driven Screen Marker Calibration" self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()] self.menu.append(ui.Info_Text("Calibrate gaze parameters pressing space key for each marker.")) self.menu.append(ui.Selector('monitor_idx',self,selection = range(len(self.monitor_names)),labels=self.monitor_names,label='Monitor')) self.menu.append(ui.Switch('fullscreen',self,label='Use fullscreen')) self.menu.append(ui.Slider('marker_scale',self,step=0.1,min=0.5,max=2.0,label='Marker size')) self.menu.append(ui.Slider('sample_duration',self,step=1,min=10,max=100,label='Sample duration')) def start(self): if not self.g_pool.capture.online: logger.error("{} requires world capture video input.".format(self.mode_pretty)) return super().start() logger.info("Starting {}".format(self.mode_pretty)) if self.mode == 'calibration': self.sites = [(.5, .5), (.25, .5), (0, .5), (1., .5), (.75, .5), (.5, 1.), (.25, 1.), (0., 1.), (1., 1.), (.75, 1.), (.25, .0), (1., 0.), (.5, 0.), (0., 0.), (.75, .0) ] shuffle(self.sites) else: self.sites = [(.5, .5), (.25, .25), (.25, .75), (.75, .75), (.75, .25)] self.active_site = self.sites.pop(0) self.active = True self.ref_list = [] self.pupil_list = [] self.clicks_to_close = 5 self.open_window(self.mode_pretty) def open_window(self,title='new_window'): if not self._window: if self.fullscreen: monitor = glfwGetMonitors()[self.monitor_idx] width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(monitor) else: monitor = None width,height= 640,360 self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext()) if not self.fullscreen: glfwSetWindowPos(self._window, self.window_position_default[0], self.window_position_default[1]) glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN) # Register callbacks glfwSetFramebufferSizeCallback(self._window, on_resize) glfwSetKeyCallback(self._window, self.on_window_key) glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button) on_resize(self._window, *glfwGetFramebufferSize(self._window)) # gl_state settings active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) def on_window_key(self,window, key, scancode, action, mods): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: self.clicks_to_close = 0 if key == GLFW_KEY_SPACE: self.space_key_was_pressed = True def on_window_mouse_button(self,window,button, action, mods): if action ==GLFW_PRESS: self.clicks_to_close -=1 def stop(self): # TODO: redundancy between all gaze mappers -> might be moved to parent class logger.info("Stopping {}".format(self.mode_pretty)) self.smooth_pos = 0, 0 self.counter = 0 self.close_window() self.active = False self.button.status_text = '' if self.mode == 'calibration': finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == 'accuracy_test': self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def close_window(self): if self._window: # enable mouse display active_window = glfwGetCurrentContext(); glfwSetInputMode(self._window,GLFW_CURSOR,GLFW_CURSOR_NORMAL) glfwDestroyWindow(self._window) self._window = None glfwMakeContextCurrent(active_window) def recent_events(self, events): frame = events.get('frame') if self.active and frame: recent_pupil_positions = events['pupil_positions'] gray_img = frame.gray if self.clicks_to_close <=0: self.stop() return # Update the marker self.markers = self.circle_tracker.update(gray_img) # Screen marker takes only Ref marker self.markers = [marker for marker in self.markers if marker['marker_type'] == 'Ref'] if len(self.markers) > 0: self.detected = True # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]['img_pos'] self.pos = self.markers[0]['norm_pos'] else: self.detected = False self.pos = None # indicate that no reference is detected if len(self.markers) > 1: logger.warning("{} markers detected. Please remove all the other markers".format(len(self.markers))) # only save a valid ref position if within sample window of calibraiton routine on_position = self.lead_in < self.screen_marker_state < (self.lead_in+self.sample_duration) if on_position and self.detected and self.space_key_was_pressed: ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # always save pupil positions for p_pt in recent_pupil_positions: if p_pt['confidence'] > self.pupil_confidence_threshold: self.pupil_list.append(p_pt) if on_position and self.detected and events.get('fixations', []) and self.space_key_was_pressed: self.screen_marker_state = min( self.sample_duration+self.lead_in, self.screen_marker_state+self.fixation_boost) # Animate the screen marker if self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out: if (self.detected and self.space_key_was_pressed) or not on_position: self.screen_marker_state += 1 else: self.space_key_was_pressed = False self.screen_marker_state = 0 if not self.sites: self.stop() return self.active_site = self.sites.pop(0) logger.debug("Moving screen marker to site at {} {}".format(*self.active_site)) # use np.arrays for per element wise math self.display_pos = np.array(self.active_site) self.on_position = on_position self.button.status_text = '{} / {}'.format(self.active_site, 15) if self._window: self.gl_display_in_window() def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active and self.detected: for marker in self.markers: e = marker['ellipses'][-1] # outermost ellipse pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0]/2), int(e[1][1]/2)), int(e[-1]), 0, 360, 15) draw_polyline(pts, 1, RGBA(0.,1.,0.,1.)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1., 0., 0., .5), line_type=gl.GL_POLYGON) def gl_display_in_window(self): active_window = glfwGetCurrentContext() if glfwWindowShouldClose(self._window): self.close_window() return glfwMakeContextCurrent(self._window) clear_gl_screen() hdpi_factor = glfwGetFramebufferSize(self._window)[0]/glfwGetWindowSize(self._window)[0] r = self.marker_scale * hdpi_factor gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() p_window_size = glfwGetFramebufferSize(self._window) gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1) # Switch back to Model View Matrix gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() def map_value(value,in_range=(0,1),out_range=(0,1)): ratio = (out_range[1]-out_range[0])/(in_range[1]-in_range[0]) return (value-in_range[0])*ratio+out_range[0] pad = 90 * r screen_pos = map_value(self.display_pos[0],out_range=(pad,p_window_size[0]-pad)),map_value(self.display_pos[1],out_range=(p_window_size[1]-pad,pad)) alpha = interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in)) r2 = 2 * r draw_points([screen_pos], size=60*r2, color=RGBA(0., 0., 0., alpha), sharpness=0.9) draw_points([screen_pos], size=38*r2, color=RGBA(1., 1., 1., alpha), sharpness=0.8) draw_points([screen_pos], size=19*r2, color=RGBA(0., 0., 0., alpha), sharpness=0.55) # some feedback on the detection state and button pressing if self.detected and self.on_position and self.space_key_was_pressed: color = RGBA(.8,.8,0., alpha) else: if self.detected: color = RGBA(0.,.8,0., alpha) else: color = RGBA(.8,0.,0., alpha) draw_points([screen_pos],size=3*r2,color=color,sharpness=0.5) if self.clicks_to_close <5: self.glfont.set_size(int(p_window_size[0]/30.)) self.glfont.draw_text(p_window_size[0]/2.,p_window_size[1]/4.,'Touch {} more times to cancel {}.'.format(self.clicks_to_close, self.mode_pretty)) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) def get_init_dict(self): d = {} d['fullscreen'] = self.fullscreen d['marker_scale'] = self.marker_scale d['sample_duration'] = self.sample_duration d['monitor_idx'] = self.monitor_idx return d def deinit_ui(self): """gets called when the plugin get terminated. either voluntarily or forced. """ if self.active: self.stop() if self._window: self.close_window() super().deinit_ui()
class Screen_Marker_Calibration(Calibration_Plugin): """Calibrate using a marker on your screen We use a ring detector that moves across the screen to 9 sites Points are collected at sites - not between """ def __init__(self, g_pool, fullscreen=True, marker_scale=1.0, sample_duration=500, monitor_idx=0): super().__init__(g_pool) self.screen_marker_state = 0. self.sample_duration = sample_duration # number of frames to sample per site self.lead_in = 100 # frames of marker shown before starting to sample self.lead_out = 15 # frames of markers shown after sampling is donw self.active_site = None self.sites = [] self.display_pos = -1., -1. self.on_position = False self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.monitor_idx = monitor_idx self.fullscreen = fullscreen self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font('opensans',get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2,0.5,0.9,1.0)) self.glfont.set_align_string(v_align='center') self.window_position_default = (0, 0) self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.menu.label = "Screen Marker Calibration with Jerk Nystagmus" def get_monitors_idx_list(): monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()] return range(len(monitors)),monitors if self.monitor_idx not in get_monitors_idx_list()[0]: logger.warning("Monitor at index %s no longer availalbe using default"%self.monitor_idx) self.monitor_idx = 0 self.menu.append(ui.Info_Text("Calibrate gaze parameters using a screen based animation.")) self.menu.append(ui.Selector('monitor_idx',self,selection_getter = get_monitors_idx_list,label='Monitor')) self.menu.append(ui.Switch('fullscreen',self,label='Use fullscreen')) self.menu.append(ui.Slider('marker_scale',self,step=0.1,min=0.5,max=2.0,label='Marker size')) self.menu.append(ui.Slider('sample_duration',self,step=1,min=10,max=100,label='Sample duration')) def start(self): if not self.g_pool.capture.online: logger.error("{} requiers world capture video input.".format(self.mode_pretty)) return super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) if self.mode == 'calibration': self.sites = [(.25, .5), (0, .5), (0., 1.), (.5, 1.), (1., 1.), (1., .5), (1., 0.), (.5, 0.), (0., 0.), (.75, .5)] else: self.sites = [(.5, .5), (.25, .25), (.25, .75), (.75, .75), (.75, .25)] self.active_site = self.sites.pop(0) self.active = True self.ref_list = [] self.pupil_list = [] self.clicks_to_close = 5 self.open_window(self.mode_pretty) def open_window(self, title='new_window'): if not self._window: if self.fullscreen: try: monitor = glfwGetMonitors()[self.monitor_idx] except: logger.warning("Monitor at index %s no longer availalbe using default"%self.monitor_idx) self.monitor_idx = 0 monitor = glfwGetMonitors()[self.monitor_idx] width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(monitor) else: monitor = None width,height= 640,360 self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext()) if not self.fullscreen: glfwSetWindowPos(self._window, self.window_position_default[0], self.window_position_default[1]) glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN) # Register callbacks glfwSetFramebufferSizeCallback(self._window, on_resize) glfwSetKeyCallback(self._window, self.on_window_key) glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button) on_resize(self._window, *glfwGetFramebufferSize(self._window)) # gl_state settings active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) def on_window_key(self,window, key, scancode, action, mods): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: self.clicks_to_close = 0 def on_window_mouse_button(self,window,button, action, mods): if action ==GLFW_PRESS: self.clicks_to_close -=1 def stop(self): # TODO: redundancy between all gaze mappers -> might be moved to parent class audio.say("Stopping {}".format(self.mode_pretty)) logger.info("Stopping {}".format(self.mode_pretty)) self.smooth_pos = 0, 0 self.counter = 0 self.close_window() self.active = False self.button.status_text = '' fixation_list = is_fixation(self) if self.mode == 'calibration': finish_calibration(self.g_pool, self.fixation_list, self.ref_list) elif self.mode == 'accuracy_test': self.finish_accuracy_test(self.fixation_list, self.ref_list) super().stop() def close_window(self): if self._window: # enable mouse display active_window = glfwGetCurrentContext() glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL) glfwDestroyWindow(self._window) self._window = None glfwMakeContextCurrent(active_window) def recent_events(self, events): frame = events.get('frame') if self.active and frame: gray_img = frame.gray if self.clicks_to_close <=0: self.stop() return # Update the marker self.markers = self.circle_tracker.update(gray_img) # Screen marker takes only Ref marker self.markers = [marker for marker in self.markers if marker['marker_type'] == 'Ref'] if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]['img_pos'] self.pos = self.markers[0]['norm_pos'] else: self.pos = None # indicate that no reference is detected # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning("{} markers detected. Please remove all the other markers".format(len(self.markers))) # only save a valid ref position if within sample window of calibration routine on_position = self.lead_in < self.screen_marker_state < (self.lead_in+self.sample_duration) if on_position and len(self.markers): ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # Always save pupil positions self.pupil_list.extend(events['pupil_positions']) ''' if on_position and len(self.markers) and events.get('fixations', []): fixation_boost = 5 self.screen_marker_state = min( self.sample_duration+self.lead_in, self.screen_marker_state+fixation_boost) ''' # Animate the screen marker if self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out: if len(self.markers) or not on_position: self.screen_marker_state += 1 else: self.screen_marker_state = 0 if not self.sites: self.stop() return self.active_site = self.sites.pop(0) logger.debug("Moving screen marker to site at {} {}".format(*self.active_site)) # use np.arrays for per element wise math self.display_pos = np.array(self.active_site) self.on_position = on_position self.button.status_text = '{}'.format(self.active_site) if self._window: self.gl_display_in_window()
class Single_Marker_Calibration(Calibration_Plugin): """Calibrate using a single marker. Move your head for example in a spiral motion while gazing at the marker to quickly sample a wide range gaze angles. """ def __init__( self, g_pool, marker_mode="Full screen", marker_scale=1.0, sample_duration=40, monitor_idx=0, ): super().__init__(g_pool) self.screen_marker_state = 0.0 self.lead_in = 25 # frames of marker shown before starting to sample self.display_pos = (0.5, 0.5) self.on_position = False self.pos = None self.marker_scale = marker_scale self._window = None self.menu = None self.stop_marker_found = False self.auto_stop = 0 self.auto_stop_max = 30 self.monitor_idx = monitor_idx self.marker_mode = marker_mode self.clicks_to_close = 5 self.glfont = fontstash.Context() self.glfont.add_font("opensans", get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0)) self.glfont.set_align_string(v_align="center") # UI Platform tweaks if system() == "Linux": self.window_position_default = (0, 0) elif system() == "Windows": self.window_position_default = (8, 90) else: self.window_position_default = (0, 0) self.circle_tracker = CircleTracker() self.markers = [] def init_ui(self): super().init_ui() self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()] def get_monitors_idx_list(): monitors = [glfwGetMonitorName(m) for m in glfwGetMonitors()] return range(len(monitors)), monitors if self.monitor_idx not in get_monitors_idx_list()[0]: logger.warning( "Monitor at index %s no longer availalbe using default" % idx ) self.monitor_idx = 0 self.menu.append( ui.Info_Text( "Calibrate using a single marker. Gaze at the center of the marker and move your head (e.g. in a slow spiral movement). This calibration method enables you to quickly sample a wide range of gaze angles and cover a large range of your FOV." ) ) self.menu.append( ui.Selector( "marker_mode", self, selection=["Full screen", "Window", "Manual"], label="Marker display mode", ) ) self.menu.append( ui.Selector( "monitor_idx", self, selection_getter=get_monitors_idx_list, label="Monitor", ) ) self.menu.append( ui.Slider( "marker_scale", self, step=0.1, min=0.5, max=2.0, label="Marker size" ) ) def start(self): if not self.g_pool.capture.online: logger.error("This calibration requires world capture video input.") return super().start() audio.say("Starting {}".format(self.mode_pretty)) logger.info("Starting {}".format(self.mode_pretty)) self.active = True self.ref_list = [] self.pupil_list = [] self.clicks_to_close = 5 if self.marker_mode != "Manual": self.open_window(self.mode_pretty) def open_window(self, title="new_window"): if not self._window: if self.marker_mode == "Full screen": try: monitor = glfwGetMonitors()[self.monitor_idx] except: logger.warning( "Monitor at index %s no longer availalbe using default" % idx ) self.monitor_idx = 0 monitor = glfwGetMonitors()[self.monitor_idx] width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode( monitor ) else: monitor = None width, height = 640, 360 self._window = glfwCreateWindow( width, height, title, monitor=monitor, share=glfwGetCurrentContext() ) if self.marker_mode == "Window": glfwSetWindowPos( self._window, self.window_position_default[0], self.window_position_default[1], ) glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN) # Register callbacks glfwSetFramebufferSizeCallback(self._window, on_resize) glfwSetKeyCallback(self._window, self.on_window_key) glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button) on_resize(self._window, *glfwGetFramebufferSize(self._window)) # gl_state settings active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() # refresh speed settings glfwSwapInterval(0) glfwMakeContextCurrent(active_window) def on_window_key(self, window, key, scancode, action, mods): if action == GLFW_PRESS: if self.mode == "calibration": target_key = GLFW_KEY_C else: target_key = GLFW_KEY_T if key == GLFW_KEY_ESCAPE or key == target_key: self.clicks_to_close = 0 def on_window_mouse_button(self, window, button, action, mods): if action == GLFW_PRESS: self.clicks_to_close -= 1 def stop(self): # TODO: redundancy between all gaze mappers -> might be moved to parent class audio.say("Stopping {}".format(self.mode_pretty)) logger.info("Stopping {}".format(self.mode_pretty)) self.smooth_pos = 0, 0 self.counter = 0 self.close_window() self.active = False self.button.status_text = "" if self.mode == "calibration": finish_calibration(self.g_pool, self.pupil_list, self.ref_list) elif self.mode == "accuracy_test": self.finish_accuracy_test(self.pupil_list, self.ref_list) super().stop() def close_window(self): if self._window: # enable mouse display active_window = glfwGetCurrentContext() glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_NORMAL) glfwDestroyWindow(self._window) self._window = None glfwMakeContextCurrent(active_window) def recent_events(self, events): frame = events.get("frame") if self.active and frame: gray_img = frame.gray if self.clicks_to_close <= 0: self.stop() return # Update the marker self.markers = self.circle_tracker.update(gray_img) self.stop_marker_found = False if len(self.markers): # Set the pos to be the center of the first detected marker marker_pos = self.markers[0]["img_pos"] self.pos = self.markers[0]["norm_pos"] # Check if there are stop markers for marker in self.markers: if marker["marker_type"] == "Stop": self.auto_stop += 1 self.stop_marker_found = True break else: self.pos = None # indicate that no reference is detected if self.stop_marker_found is False: self.auto_stop = 0 # Check if there are more than one markers if len(self.markers) > 1: audio.tink() logger.warning( "{} markers detected. Please remove all the other markers".format( len(self.markers) ) ) # only save a valid ref position if within sample window of calibraiton routine on_position = self.lead_in < self.screen_marker_state if on_position and len(self.markers) and not self.stop_marker_found: ref = {} ref["norm_pos"] = self.pos ref["screen_pos"] = marker_pos ref["timestamp"] = frame.timestamp self.ref_list.append(ref) # always save pupil positions self.pupil_list.extend(events["pupil"]) # Animate the screen marker if len(self.markers) or not on_position: self.screen_marker_state += 1 # Stop if autostop condition is satisfied: if self.auto_stop >= self.auto_stop_max: self.auto_stop = 0 self.stop() # use np.arrays for per element wise math self.on_position = on_position if self._window: self.gl_display_in_window() def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: # draw the largest ellipse of all detected markers for marker in self.markers: e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0)) if len(self.markers) > 1: draw_polyline( pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON ) # draw indicator on the stop marker(s) if self.auto_stop: for marker in self.markers: if marker["marker_type"] == "Stop": e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.auto_stop_max, ) indicator = [e[0]] + pts[self.auto_stop :].tolist() + [e[0]] draw_polyline( indicator, color=RGBA(8.0, 0.1, 0.1, 0.8), line_type=gl.GL_POLYGON, ) def gl_display_in_window(self): active_window = glfwGetCurrentContext() if glfwWindowShouldClose(self._window): self.close_window() return glfwMakeContextCurrent(self._window) clear_gl_screen() hdpi_factor = getHDPIFactor(self._window) r = self.marker_scale * hdpi_factor gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() p_window_size = glfwGetFramebufferSize(self._window) gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1) # Switch back to Model View Matrix gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() def map_value(value, in_range=(0, 1), out_range=(0, 1)): ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0]) return (value - in_range[0]) * ratio + out_range[0] pad = 90 * r screen_pos = ( map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)), map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)), ) alpha = ( 1.0 ) # interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in)) r2 = 2 * r draw_points( [screen_pos], size=60 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.9 ) draw_points( [screen_pos], size=38 * r2, color=RGBA(1.0, 1.0, 1.0, alpha), sharpness=0.8 ) draw_points( [screen_pos], size=19 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.55 ) # some feedback on the detection state color = ( RGBA(0.0, 0.8, 0.0, alpha) if len(self.markers) and self.on_position else RGBA(0.8, 0.0, 0.0, alpha) ) draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5) if self.clicks_to_close < 5: self.glfont.set_size(int(p_window_size[0] / 30.0)) self.glfont.draw_text( p_window_size[0] / 2.0, p_window_size[1] / 4.0, "Touch {} more times to cancel calibration.".format( self.clicks_to_close ), ) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) def get_init_dict(self): d = {} d["marker_mode"] = self.marker_mode d["marker_scale"] = self.marker_scale d["monitor_idx"] = self.monitor_idx return d def deinit_ui(self): """gets called when the plugin get terminated. either voluntarily or forced. """ if self.active: self.stop() if self._window: self.close_window() super().deinit_ui()
files.sort() for file in files: if os.path.splitext(file)[-1] != '.jpg': continue if "2017_11_30-002-world-frame-00000" not in file: continue photo_name = os.path.splitext(file)[0] photo_path = os.path.join(root, file) image = cv2.imread(photo_path) img_size = image.shape[::-1][1] gray_img = cv2.imread(photo_path, 0) print(photo_name) image_count += 1 start_time = timeit.default_timer() current_markers = circle_tracker.update(gray_img) end_time = timeit.default_timer() duration.append(end_time - start_time) if len(current_markers): for i in range(len(current_markers)): marker_pos = current_markers[i]['img_pos'][0], current_markers[i]['img_pos'][1] s = current_markers[i]['ellipses'][-1][1][0], current_markers[i]['ellipses'][-1][1][1] print(current_markers[i]['marker_type'], "marker found. Pos =", marker_pos, s) else: print("No marker found") if photo_record_mode: for i in range(len(current_markers)): marker_pos = int(current_markers[i]['img_pos'][0]), int(current_markers[i]['img_pos'][1]) color = (0, 0, 255) if current_markers[i]['marker_type'] == 'Stop' else (0, 255, 0)