def service( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, debug, ): """Maps pupil to gaze data, can run various plug-ins. Reacts to notifications: ``start_plugin``: Starts given plugin with the given arguments ``eye_process.started``: Sets the detection method eye process ``service_process.should_stop``: Stops the service process Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``service_process.started`` ``service_process.stopped`` ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the service process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("pupil", )) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) poller = zmq.Poller() poller.register(pupil_sub.socket) poller.register(notify_sub.socket) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start", "eye_id": eye_id, "delay": delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {"subject": "eye_process.should_stop", "eye_id": eye_id} ipc_pub.notify(n) try: # helpers/utils from file_methods import Persistent_Dict from methods import delta_t, get_system_info from version_utils import VersionFormat import audio from uvc import get_time_monotonic # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_choreography import ( available_calibration_choreography_plugins, patch_loaded_plugins_with_choreography_plugin, ) from gaze_mapping import registered_gazer_classes from network_api import NetworkApiPlugin from pupil_groups import Pupil_Groups from blink_detection import Blink_Detection from fixation_detector import Fixation_Detector from service_ui import Service_UI from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") # g_pool holds variables for this process they are accesible to all plugins g_pool = SimpleNamespace() g_pool.debug = debug g_pool.app = "service" g_pool.user_dir = user_dir g_pool.version = version g_pool.get_now = get_time_monotonic g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.timebase = timebase g_pool.preferred_remote_port = preferred_remote_port g_pool.hide_ui = hide_ui def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins")) user_launchable_plugins = [ Service_UI, Pupil_Groups, NetworkApiPlugin, Blink_Detection, ] + runtime_plugins plugin_by_index = (runtime_plugins + available_calibration_choreography_plugins() + registered_gazer_classes() + user_launchable_plugins) name_by_index = [ pupil_datum.__name__ for pupil_datum in plugin_by_index ] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [ ("Service_UI", {}), # Calibration choreography plugin is added bellow by calling # patch_world_session_settings_with_choreography_plugin ("NetworkApiPlugin", {}), ("Blink_Detection", {}), ("Fixation_Detector", {}), ] g_pool.plugin_by_name = plugin_by_name tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_service")) if session_settings.get("version", VersionFormat("0.0")) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) audio.set_audio_mode( session_settings.get("audio_mode", audio.get_default_audio_mode())) ipc_pub.notify({"subject": "service_process.started"}) logger.warning("Process started.") g_pool.service_should_run = True loaded_plugins = session_settings.get("loaded_plugins", default_plugins) # Resolve the active calibration choreography plugin loaded_plugins = patch_loaded_plugins_with_choreography_plugin( loaded_plugins, app=g_pool.app) session_settings["loaded_plugins"] = loaded_plugins # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, loaded_plugins) # NOTE: The NetworkApiPlugin plugin fails to load when the port is already in use # and will set this variable to false. Then we should not even start the eye # processes. Otherwise we would have to wait for their initialization before # attempting cleanup in Service. if g_pool.service_should_run: if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.3) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.0) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add(plugin_by_name[n["name"]], args=n.get("args", {})) elif subject == "service_process.should_stop": g_pool.service_should_run = False elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": service.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) # initiate ui update loop ipc_pub.notify({ "subject": "service_process.ui.should_update", "initial_delay": 1 / 40 }) g_pool.active_gaze_mapping_plugin = None # Event loop while g_pool.service_should_run and not process_was_interrupted: socks = dict(poller.poll()) if pupil_sub.socket in socks: topic, pupil_datum = pupil_sub.recv() events = {} events["pupil"] = [pupil_datum] if g_pool.active_gaze_mapping_plugin: new_gaze_data = g_pool.active_gaze_mapping_plugin.map_pupil_to_gaze( [pupil_datum]) events["gaze"] = [] for gaze_datum in new_gaze_data: gaze_pub.send(gaze_datum) events["gaze"].append(gaze_datum) for plugin in g_pool.plugins: plugin.recent_events(events=events) if notify_sub.socket in socks: topic, n = notify_sub.recv() handle_notifications(n) for plugin in g_pool.plugins: plugin.on_notify(n) # check if a plugin need to be destroyed g_pool.plugins.clean() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings["audio_mode"] = audio.get_audio_mode() session_settings.close() # de-init all running plugins for pupil_datum in g_pool.plugins: pupil_datum.alive = False g_pool.plugins.clean() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Service crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "service_process.stopped"}) # shut down launcher n = {"subject": "launcher_process.should_stop"} ipc_pub.notify(n) sleep(1.0)
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, debug, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``eye_process.started`` ``start_plugin`` ``should_stop`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def detection_enabled_getter() -> int: return int(g_pool.pupil_detection_enabled) def detection_enabled_setter(value: int): is_on = bool(value) g_pool.pupil_detection_enabled = is_on n = {"subject": "pupil_detector.set_enabled", "value": is_on} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw from gl_utils import GLFWErrorReporting GLFWErrorReporting.set_default() from version_utils import parse_version from pyglui import ui, cygl, __version__ as pyglui_version assert parse_version(pyglui_version) >= parse_version( "1.27"), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") import audio # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_choreography import ( available_calibration_choreography_plugins, CalibrationChoreographyPlugin, patch_loaded_plugins_with_choreography_plugin, ) available_choreography_plugins = available_calibration_choreography_plugins( ) from gaze_mapping import registered_gazer_classes from gaze_mapping.gazer_base import GazerBase from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from network_api import NetworkApiPlugin from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from accuracy_visualizer import Accuracy_Visualizer from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) icon_bar_width = 50 window_size = None camera_render_size = None content_scale = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.debug = debug g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins")) runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) ] user_plugins = [ Pupil_Groups, NetworkApiPlugin, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ([ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes) plugins = (system_plugins + user_plugins + runtime_plugins + available_choreography_plugins + registered_gazer_classes()) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, CalibrationChoreographyPlugin, GazerBase, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ (default_capture_name, default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), # Calibration choreography plugin is added below by calling # patch_loaded_plugins_with_choreography_plugin ("Recorder", {}), ("NetworkApiPlugin", {}), ("Fixation_Detector", {}), ("Blink_Detection", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] def consume_events_and_render_buffer(): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal content_scale if w == 0 or h == 0: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing with gl_utils.current_context(main_window): consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for plugin in g_pool.plugins: if plugin.on_drop(paths): break tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world")) if parse_version(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_data_confidence = 0.6 g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) g_pool.pupil_detection_enabled = bool( session_settings.get("pupil_detection_enabled", True)) g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.set_audio_mode( session_settings.get("audio_mode", audio.get_default_audio_mode())) def handle_notifications(noti): subject = noti["subject"] if subject == "pupil_detector.set_enabled": g_pool.pupil_detection_enabled = noti["value"] elif subject == "start_plugin": try: g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "pupil_detector.set_enabled", "value": g_pool.pupil_detection_enabled, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) elif subject == "world_process.adapt_window_size": set_window_size() elif subject == "world_process.should_stop": glfw.set_window_should_close(main_window, True) width, height = session_settings.get("window_size", (1280 + icon_bar_width, 720)) # window and gl setup glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window main_window = glfw.create_window(width, height, "Pupil Capture - World", None, None) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.set_window_should_close(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "world_process.should_start", "delay": 2.0 }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "Audio mode", None, getter=audio.get_audio_mode, setter=audio.set_audio_mode, selection=audio.get_audio_mode_list(), )) general_settings.append( ui.Switch( "pupil_detection_enabled", label="Pupil detection", getter=detection_enabled_getter, setter=detection_enabled_setter, )) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, )) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, )) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version))) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) loaded_plugins = session_settings.get("loaded_plugins", default_plugins) # Resolve the active calibration choreography plugin loaded_plugins = patch_loaded_plugins_with_choreography_plugin( loaded_plugins, app=g_pool.app) session_settings["loaded_plugins"] = loaded_plugins # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, loaded_plugins) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add(g_pool.plugin_by_name[default_capture_name], default_capture_settings) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes g_pool.trigger_main_window_redraw = lambda: on_resize( main_window, *glfw.get_framebuffer_size(main_window)) g_pool.trigger_main_window_redraw() if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") if platform.system() == "Darwin": # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of # occluded windows, based on the swap interval value. This causes an FPS drop # and leads to problems when recording. To side-step this behaviour, the swap # interval is set to zero. # # Read more about window occlusion on macOS here: # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html glfw.swap_interval(0) # Event loop while not glfw.window_should_close( main_window) and not process_was_interrupted: # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.make_context_current(main_window) # render visual feedback from loaded plugins glfw.poll_events() if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings[ "pupil_detection_enabled"] = g_pool.pupil_detection_enabled session_settings["audio_mode"] = audio.get_audio_mode() if not hide_ui: glfw.restore_window(main_window) # need to do this for windows os session_settings["window_position"] = glfw.get_window_pos( main_window) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.destroy_window(main_window) glfw.terminate() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def supported_gazer_classes(cls): gazers = registered_gazer_classes() # By default, HMD gazers are not supported by regular choreographies gazers = [g for g in gazers if not issubclass(g, GazerHMD3D)] return gazers