def run(self): self._callFn("appStarted", self) def sizeChanged(window, width, height): self.width = width self.height = height print(f'new size: {width} {height}') self.canvas.resize(width, height) self._callFn('sizeChanged', self) def charEvent(*args): self.dispatchCharEvent(*args) def keyEvent(*args): self.dispatchKeyEvent(*args) def mouseMoved(*args): self.dispatchMouseMoved(*args) def mouseChanged(*args): self.dispatchMouseChanged(*args) glfw.set_framebuffer_size_callback(self.window, sizeChanged) glfw.set_key_callback(self.window, keyEvent) glfw.set_char_callback(self.window, charEvent) glfw.set_cursor_pos_callback(self.window, mouseMoved) glfw.set_mouse_button_callback(self.window, mouseChanged) i = 0 times = [0.0] * 20 while not glfw.window_should_close(self.window): times.pop(0) glClearColor(0.2, 0.3, 0.3, 1.0) glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) #type:ignore self._callFn('redrawAll', self, self.window, self.canvas) self.canvas.redraw() if time.time() - self.lastTimer > (self.timerDelay / 1000.0): self.lastTimer = time.time() self._callFn('timerFired', self) glfw.swap_buffers(self.window) glfw.poll_events() times.append(time.time()) i += 1 if i % 50 == 0: timeDiff = (times[-1] - times[0]) / (len(times) - 1) print(timeDiff) self._callFn('appStopped', self) glfw.terminate()
def __init__(self, **kwargs): super().__init__(**kwargs) if not glfw.init(): raise ValueError("Failed to initialize glfw") # Configure the OpenGL context glfw.window_hint(glfw.CONTEXT_CREATION_API, glfw.NATIVE_CONTEXT_API) glfw.window_hint(glfw.CLIENT_API, glfw.OPENGL_API) glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, self.gl_version[0]) glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, self.gl_version[1]) glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, True) glfw.window_hint(glfw.RESIZABLE, self.resizable) glfw.window_hint(glfw.DOUBLEBUFFER, True) glfw.window_hint(glfw.DEPTH_BITS, 24) glfw.window_hint(glfw.SAMPLES, self.samples) glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) monitor = None if self.fullscreen: self._set_fullscreen(True) self._window = glfw.create_window(self.width, self.height, self.title, monitor, None) self._has_focus = True if not self._window: glfw.terminate() raise ValueError("Failed to create window") self.cursor = self._cursor self._buffer_width, self._buffer_height = glfw.get_framebuffer_size( self._window) glfw.make_context_current(self._window) if self.vsync: glfw.swap_interval(1) else: glfw.swap_interval(0) glfw.set_key_callback(self._window, self.glfw_key_event_callback) glfw.set_cursor_pos_callback(self._window, self.glfw_mouse_event_callback) glfw.set_mouse_button_callback(self._window, self.glfw_mouse_button_callback) glfw.set_scroll_callback(self._window, self.glfw_mouse_scroll_callback) glfw.set_window_size_callback(self._window, self.glfw_window_resize_callback) glfw.set_char_callback(self._window, self.glfw_char_callback) glfw.set_window_focus_callback(self._window, self.glfw_window_focus) glfw.set_cursor_enter_callback(self._window, self.glfw_cursor_enter) glfw.set_window_iconify_callback(self._window, self.glfw_window_iconify) glfw.set_window_close_callback(self._window, self.glfw_window_close) self.init_mgl_context() self.set_default_viewport()
def set_callbacks(self): glfw.set_window_close_callback(self.window, self.window_close_callback) glfw.set_window_size_callback(self.window, self.window_resize_callback) glfw.set_key_callback(self.window, self.keyboard_callback) glfw.set_char_callback(self.window, self.key_typed_callback) glfw.set_mouse_button_callback(self.window, self.mouse_callback) glfw.set_scroll_callback(self.window, self.scroll_callback) glfw.set_cursor_pos_callback(self.window, self.cursor_pos_callback)
def __init__(self, **kwargs): super().__init__(**kwargs) if not glfw.init(): raise ValueError("Failed to initialize glfw") # Configure the OpenGL context glfw.window_hint(glfw.CONTEXT_CREATION_API, glfw.NATIVE_CONTEXT_API) glfw.window_hint(glfw.CLIENT_API, glfw.OPENGL_API) glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, self.gl_version[0]) glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, self.gl_version[1]) glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, True) glfw.window_hint(glfw.RESIZABLE, self.resizable) glfw.window_hint(glfw.DOUBLEBUFFER, True) glfw.window_hint(glfw.DEPTH_BITS, 24) glfw.window_hint(glfw.SAMPLES, self.samples) monitor = None if self.fullscreen: monitor = glfw.get_primary_monitor() mode = glfw.get_video_mode(monitor) self._width, self._height = mode.size.width, mode.size.height glfw.window_hint(glfw.RED_BITS, mode.bits.red) glfw.window_hint(glfw.GREEN_BITS, mode.bits.green) glfw.window_hint(glfw.BLUE_BITS, mode.bits.blue) glfw.window_hint(glfw.REFRESH_RATE, mode.refresh_rate) self._window = glfw.create_window(self.width, self.height, self.title, monitor, None) if not self._window: glfw.terminate() raise ValueError("Failed to create window") if not self.cursor: glfw.set_input_mode(self._window, glfw.CURSOR, glfw.CURSOR_DISABLED) self._buffer_width, self._buffer_height = glfw.get_framebuffer_size(self._window) glfw.make_context_current(self._window) if self.vsync: glfw.swap_interval(1) glfw.set_key_callback(self._window, self.glfw_key_event_callback) glfw.set_cursor_pos_callback(self._window, self.glfw_mouse_event_callback) glfw.set_mouse_button_callback(self._window, self.glfw_mouse_button_callback) glfw.set_scroll_callback(self._window, self.glfw_mouse_scroll_callback) glfw.set_window_size_callback(self._window, self.glfw_window_resize_callback) glfw.set_char_callback(self._window, self.glfw_char_callback) self.init_mgl_context() self.set_default_viewport()
def wire_events(self, gl, window): """ :param gl: moderngl context :param window: glfw window handle """ def on_resize(window, width, height): gl.viewport = (0, 0, width, height) imgui.get_io().display_size = width, height self.on_resize(window, width, height) def on_cursor_pos(window, x, y): imgui.get_io().mouse_pos = x, y self.on_cursor_pos(window, x, y) def on_mouse_button(window, button, action, mods): imgui.get_io().mouse_down[button] = action self.on_mouse_button(window, button, action, mods) def on_scroll(window, scroll_x, scroll_y): imgui.get_io().mouse_wheel = scroll_y self.on_scroll(window, scroll_x, scroll_y) def on_key(window, key, scancode, action, mods): io = self.io if action == glfw.PRESS: io.keys_down[key] = True elif action == glfw.RELEASE: io.keys_down[key] = False io.key_ctrl = (io.keys_down[glfw.KEY_LEFT_CONTROL] or io.keys_down[glfw.KEY_RIGHT_CONTROL]) io.key_alt = (io.keys_down[glfw.KEY_LEFT_ALT] or io.keys_down[glfw.KEY_RIGHT_ALT]) io.key_shift = (io.keys_down[glfw.KEY_LEFT_SHIFT] or io.keys_down[glfw.KEY_RIGHT_SHIFT]) io.key_super = (io.keys_down[glfw.KEY_LEFT_SUPER] or io.keys_down[glfw.KEY_RIGHT_SUPER]) self.on_key(window, key, scancode, action, mods) def on_char(window, char): io = imgui.get_io() if 0 < char < 0x10000: io.add_input_character(char) glfw.set_window_size_callback(window, on_resize) glfw.set_cursor_pos_callback(window, on_cursor_pos) glfw.set_mouse_button_callback(window, on_mouse_button) glfw.set_scroll_callback(window, on_scroll) glfw.set_key_callback(window, on_key) glfw.set_char_callback(window, on_char)
def __init__( self, controller: BaseController, *, hint: Optional[GLContextHint] = None, title: str = 'glfw', width: int = 0, height: int = 0, is_maximized: bool = False, config: Optional[WindowConfig] = None ): width = width or (config.width if config else 1280) height = height or (config.height if config else 720) is_maximized = is_maximized or ( config.is_maximized if config else False) self.controller = controller self.mouse_x = 0 self.mouse_y = 0 if not glfw.init(): logger.error("Could not initialize OpenGL context") return if hint: # OS X supports only forward-compatible core profiles from 3.2 glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, hint.major) glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, hint.minor) if hint.core_profile: glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) else: glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_FORWARD_COMPAT) # Create a windowed mode window and its OpenGL context self.window = glfw.create_window( width, height, title, None, None ) if not self.window: logger.error("Could not initialize Window") return if is_maximized: glfw.maximize_window(self.window) glfw.make_context_current(self.window) glfw.swap_interval(1) glfw.set_key_callback(self.window, self.keyboard_callback) glfw.set_cursor_pos_callback(self.window, self.cursor_calblack) glfw.set_mouse_button_callback(self.window, self.mouse_callback) glfw.set_window_size_callback(self.window, self.resize_callback) glfw.set_char_callback(self.window, self.char_callback) glfw.set_scroll_callback(self.window, self.scroll_callback) self.controller.onResize(width, height)
def open_window(self): if not self.window: self.input = {"button": None, "mouse": (0, 0)} # get glfw started if self.run_independently: glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) self.window = glfw.create_window(self.window_size[0], self.window_size[1], self.name, None, None) else: self.window = glfw.create_window( self.window_size[0], self.window_size[1], self.name, None, glfw.get_current_context(), ) self.other_window = glfw.get_current_context() glfw.make_context_current(self.window) glfw.swap_interval(0) glfw.set_window_pos(self.window, window_position_default[0], window_position_default[1]) # Register callbacks window glfw.set_framebuffer_size_callback(self.window, self.on_resize) glfw.set_window_iconify_callback(self.window, self.on_iconify) glfw.set_key_callback(self.window, self.on_window_key) glfw.set_char_callback(self.window, self.on_window_char) glfw.set_mouse_button_callback(self.window, self.on_window_mouse_button) glfw.set_cursor_pos_callback(self.window, self.on_pos) glfw.set_scroll_callback(self.window, self.on_scroll) # get glfw started if self.run_independently: glutils.init() self.basic_gl_setup() self.sphere = glutils.Sphere(20) self.glfont = fs.Context() self.glfont.add_font("opensans", get_opensans_font_path()) self.glfont.set_size(18) self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0)) self.on_resize(self.window, *glfw.get_framebuffer_size(self.window)) glfw.make_context_current(self.other_window)
def __init__(self, window, attach_callbacks=True): super(GlfwRenderer, self).__init__() self.window = window if attach_callbacks: glfw.set_key_callback(self.window, self.keyboard_callback) glfw.set_cursor_pos_callback(self.window, self.mouse_callback) glfw.set_window_size_callback(self.window, self.resize_callback) glfw.set_char_callback(self.window, self.char_callback) glfw.set_scroll_callback(self.window, self.scroll_callback) self.io.display_size = glfw.get_framebuffer_size(self.window) self._map_keys() self._gui_time = None
def __init__(self, window, attach_callbacks=True): #self.io is initialized by super-class super(GlumpyRenderer, self).__init__() self.window = window if attach_callbacks: glfw.set_key_callback(self.window, self.keyboard_callback) glfw.set_cursor_pos_callback(self.window, self.mouse_callback) glfw.set_window_size_callback(self.window, self.resize_callback) glfw.set_char_callback(self.window, self.char_callback) glfw.set_scroll_callback(self.window, self.scroll_callback) self.io.display_size = glfw.get_framebuffer_size(self.window) #self.io.get_clipboard_text_fn = self._get_clipboard_text #self.io.set_clipboard_text_fn = self._set_clipboard_text self._map_keys() self._gui_time = None
def enable(self): # setup input callbacks # todo: add some option to have additional callbacks glfw.set_key_callback(self.window, self.keyboard_callback) glfw.set_cursor_pos_callback(self.window, self.mouse_callback) glfw.set_window_size_callback(self.window, self.resize_callback) glfw.set_char_callback(self.window, self.char_callback) glfw.set_scroll_callback(self.window, self.scroll_callback) # todo: maybe it is not necessary self.io.delta_time = 1.0 / 60.0 self.io.display_size = glfw.get_framebuffer_size(self.window) # setup default font self.io.fonts.get_tex_data_as_rgba32() self.io.fonts.add_font_default() self._create_device_objects() self._map_keys() # todo: add option to set new_frame callback/implementation self.io.render_callback = self.render
def __init__(self, width=640, height=480, title="Hello world"): imgui.create_context() if not glfw.init(): return self.window = glfw.create_window(width, height, title, None, None) if not self.window: glfw.terminate() return glfw.make_context_current(self.window) self.ctx = moderngl.create_context(require=460) self.impl = ImguiRenderer(self.window, attach_callbacks=False) glfw.set_key_callback(self.window, self._on_key) glfw.set_cursor_pos_callback(self.window, self._on_mouse_move) glfw.set_mouse_button_callback(self.window, self._on_mouse_button) glfw.set_window_size_callback(self.window, self._on_resize) glfw.set_char_callback(self.window, self._on_char) glfw.set_scroll_callback(self.window, self._on_scroll) self.init()
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version, debug): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw from gl_utils import GLFWErrorReporting GLFWErrorReporting.set_default() # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import parse_version from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from pupil_recording import PupilRecording from csv_utils import write_key_value_file from hotkey import Hotkey # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from seek_control import Seek_Control from surface_tracker import Surface_Tracker_Offline # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import ( DisabledPupilProducer, Pupil_From_Recording, Offline_Pupil_Detection, ) from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay from pupil_recording import ( assert_valid_recording_type, InvalidRecordingException, ) assert parse_version(pyglui_version) >= parse_version( "1.29"), "pyglui out of date, please upgrade to newest version" process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) runtime_plugins = import_runtime_plugins( os.path.join(user_dir, "plugins")) runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) ] system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, Offline_Fixation_Detector, Offline_Blink_Detection, Surface_Tracker_Offline, Raw_Data_Exporter, Annotation_Player, Log_History, DisabledPupilProducer, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, Offline_Head_Pose_Tracker, ] + runtime_plugins plugins = system_plugins + user_plugins def consume_events_and_render_buffer(): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for b in user_input.buttons: button, action, mods = b x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal content_scale if w == 0 or h == 0: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for path in paths: try: assert_valid_recording_type(path) _restart_with_recording(path) return except InvalidRecordingException as err: logger.debug(str(err)) for plugin in g_pool.plugins: if plugin.on_drop(paths): break def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": rec_dir }) glfw.set_window_should_close(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) recording = PupilRecording(rec_dir) meta_info = recording.meta_info # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") icon_bar_width = 50 window_size = None content_scale = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() g_pool.app = "player" g_pool.process = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None video_path = recording.files().core().world().videos()[0].resolve() File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if parse_version(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}" glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) main_window = glfw.create_window(width, height, window_name, None, None) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT) # populated by producers g_pool.pupil_positions = pm.PupilDataBisector() g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() g_pool.eye_movements = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info.min_player_version, "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.set_window_should_close(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu("User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline") g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Info_Text( f"Minimum Player Version: {meta_info.min_player_version}")) general_settings.append( ui.Info_Text(f"Player Version: {g_pool.version}")) general_settings.append( ui.Info_Text( f"Recording Software: {meta_info.recording_software_name}")) general_settings.append( ui.Info_Text( f"Recording Software Version: {meta_info.recording_software_version}" )) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." )) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", )) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey=Hotkey.EXPORT_START_PLAYER_HOTKEY(), label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins _pupil_producer_plugins = [ # In priority order (first is default) ("Pupil_From_Recording", {}), ("Offline_Pupil_Detection", {}), ("DisabledPupilProducer", {}), ] _pupil_producer_plugins = list(reversed(_pupil_producer_plugins)) _gaze_producer_plugins = [ # In priority order (first is default) ("GazeFromRecording", {}), ("GazeFromOfflineCalibration", {}), ] _gaze_producer_plugins = list(reversed(_gaze_producer_plugins)) default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), *_pupil_producer_plugins, *_gaze_producer_plugins, ("Audio_Playback", {}), ] _plugins_to_load = session_settings.get("loaded_plugins", None) if _plugins_to_load is None: # If no plugins are available from a previous session, # then use the default plugin list _plugins_to_load = default_plugins else: # If there are plugins available from a previous session, # then prepend plugins that are required, but might have not been available before _plugins_to_load = [ *_pupil_producer_plugins, *_gaze_producer_plugins, *_plugins_to_load, ] g_pool.plugins = Plugin_List(g_pool, _plugins_to_load) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.get_framebuffer_size(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add(g_pool.plugin_by_name[n["name"]], args=n.get("args", {})) elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) while not glfw.window_should_close( main_window) and not process_was_interrupted: # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.make_context_current(main_window) glfw.poll_events() # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for b in user_input.buttons: button, action, mods = b x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.swap_buffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.get_window_pos(main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.destroy_window(main_window) except Exception: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
#glfw.ext.set_icons([(icon_data, icon_width, icon_height)]) glfw.set_window_title(w, "pyglfw test") #glfw.disable(w, glfw.AUTO_POLL_EVENTS) #glfw.enable(w, glfw.KEY_REPEAT) center_x = int(vm[0][0] / 2 - glfw.get_window_size(w)[0] / 2) center_y = int(vm[0][1] / 2 - glfw.get_window_size(w)[1] / 2) print( "new window position: {!s}, {!s}".format(center_x, center_y) ) glfw.set_window_pos(w, center_x, center_y) glfw.set_window_size_callback(w, on_resize) glfw.set_window_close_callback(w, on_close) glfw.set_window_refresh_callback(w, on_refresh) glfw.set_key_callback(w, on_key) glfw.set_char_callback(w, on_char) glfw.set_mouse_button_callback(w, on_button) glfw.set_cursor_pos_callback(w, on_pos) glfw.set_scroll_callback(w, on_scroll) while not glfw.window_should_close(w): glfw.poll_events() if glfw.get_key(w, glfw.KEY_E) == glfw.PRESS: break glClear(GL_COLOR_BUFFER_BIT) glfw.swap_buffers(w) glfw.close_window(w) glfw.terminate()
def __init__(self, vispy_canvas, **kwargs): BaseCanvasBackend.__init__(self, vispy_canvas) p = self._process_backend_kwargs(kwargs) self._initialized = False # Deal with config _set_config(p.context.config) # Deal with context p.context.shared.add_ref('glfw', self) if p.context.shared.ref is self: share = None else: share = p.context.shared.ref._id glfw.window_hint(glfw.REFRESH_RATE, 0) # highest possible glfw.window_hint(glfw.RESIZABLE, int(p.resizable)) glfw.window_hint(glfw.DECORATED, int(p.decorate)) glfw.window_hint(glfw.VISIBLE, 0) # start out hidden glfw.window_hint(glfw.FLOATING, int(p.always_on_top)) if p.fullscreen is not False: self._fullscreen = True if p.fullscreen is True: monitor = glfw.get_primary_monitor() else: monitor = glfw.get_monitors() if p.fullscreen >= len(monitor): raise ValueError('fullscreen must be <= %s' % len(monitor)) monitor = monitor[p.fullscreen] use_size = glfw.get_video_mode(monitor)[:2] if use_size != tuple(p.size): logger.debug('Requested size %s, will be ignored to ' 'use fullscreen mode %s' % (p.size, use_size)) size = use_size else: self._fullscreen = False monitor = None size = p.size self._id = glfw.create_window(width=size[0], height=size[1], title=p.title, monitor=monitor, share=share) if not self._id: raise RuntimeError('Could not create window') glfw.make_context_current(self._id) glfw.swap_interval(1 if p.vsync else 0) # needs a valid context _VP_GLFW_ALL_WINDOWS.append(self) self._mod = list() # Register callbacks glfw.set_window_refresh_callback(self._id, self._on_draw) glfw.set_window_size_callback(self._id, self._on_resize) glfw.set_key_callback(self._id, self._on_key_press) glfw.set_char_callback(self._id, self._on_key_char) glfw.set_mouse_button_callback(self._id, self._on_mouse_button) glfw.set_scroll_callback(self._id, self._on_mouse_scroll) glfw.set_cursor_pos_callback(self._id, self._on_mouse_motion) glfw.set_window_close_callback(self._id, self._on_close) self._vispy_canvas_ = None self._needs_draw = False self._vispy_canvas.set_current() if p.position is not None: self._vispy_set_position(*p.position) if p.show: glfw.show_window(self._id) # Init self._initialized = True self._next_key_events = [] self._next_key_text = {} self._vispy_canvas.set_current() self._vispy_canvas.events.initialize() self._on_resize(self._id, size[0], size[1])
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, debug, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``eye_process.started`` ``start_plugin`` ``should_stop`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def detection_enabled_getter() -> bool: return g_pool.pupil_detection_enabled def detection_enabled_setter(is_on: bool): g_pool.pupil_detection_enabled = is_on n = {"subject": "set_pupil_detection_enabled", "value": is_on} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw glfw.ERROR_REPORTING = "raise" from version_utils import parse_version from pyglui import ui, cygl, __version__ as pyglui_version assert parse_version(pyglui_version) >= parse_version( "1.27" ), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") import audio # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_choreography import ( available_calibration_choreography_plugins, CalibrationChoreographyPlugin, patch_loaded_plugins_with_choreography_plugin, ) available_choreography_plugins = available_calibration_choreography_plugins() from gaze_mapping import registered_gazer_classes from gaze_mapping.gazer_base import GazerBase from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from network_api import NetworkApiPlugin from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from accuracy_visualizer import Accuracy_Visualizer from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) icon_bar_width = 50 window_size = None camera_render_size = None content_scale = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.debug = debug g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins") ) runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) ] user_plugins = [ Pupil_Groups, NetworkApiPlugin, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ( [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes ) plugins = ( system_plugins + user_plugins + runtime_plugins + available_choreography_plugins + registered_gazer_classes() ) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, CalibrationChoreographyPlugin, GazerBase, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ (default_capture_name, default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), # Calibration choreography plugin is added below by calling # patch_loaded_plugins_with_choreography_plugin ("Recorder", {}), ("NetworkApiPlugin", {}), ("Fixation_Detector", {}), ("Blink_Detection", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] def consume_events_and_render_buffer(): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal content_scale if w == 0 or h == 0: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None ) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for plugin in g_pool.plugins: if plugin.on_drop(paths): break tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world") ) if parse_version(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_data_confidence = 0.6 g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8 ) g_pool.pupil_detection_enabled = session_settings.get( "pupil_detection_enabled", True ) g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.set_audio_mode( session_settings.get("audio_mode", audio.get_default_audio_mode()) ) def handle_notifications(noti): subject = noti["subject"] if subject == "set_pupil_detection_enabled": g_pool.pupil_detection_enabled = noti["value"] elif subject == "start_plugin": try: g_pool.plugins.add( g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {}) ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_pupil_detection_enabled", "value": g_pool.pupil_detection_enabled, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) elif subject == "world_process.adapt_window_size": set_window_size() elif subject == "world_process.should_stop": glfw.set_window_should_close(main_window, True) width, height = session_settings.get( "window_size", (1280 + icon_bar_width, 720) ) # window and gl setup glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window main_window = glfw.create_window( width, height, "Pupil Capture - World", None, None ) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.set_window_should_close(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0}) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "Audio mode", None, getter=audio.get_audio_mode, setter=audio.set_audio_mode, selection=audio.get_audio_mode_list(), ) ) general_settings.append( ui.Switch( "pupil_detection_enabled", label="Pupil detection", getter=detection_enabled_getter, setter=detection_enabled_setter, ) ) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, ) ) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, ) ) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version)) ) general_settings.append( ui.Button("Restart with default settings", reset_restart) ) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) loaded_plugins = session_settings.get("loaded_plugins", default_plugins) # Resolve the active calibration choreography plugin loaded_plugins = patch_loaded_plugins_with_choreography_plugin( loaded_plugins, app=g_pool.app ) session_settings["loaded_plugins"] = loaded_plugins # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, loaded_plugins) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add( g_pool.plugin_by_name[default_capture_name], default_capture_settings ) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.get_framebuffer_size(main_window)) if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") if platform.system() == "Darwin": # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of # occluded windows, based on the swap interval value. This causes an FPS drop # and leads to problems when recording. To side-step this behaviour, the swap # interval is set to zero. # # Read more about window occlusion on macOS here: # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html glfw.swap_interval(0) # Event loop while not glfw.window_should_close(main_window) and not process_was_interrupted: # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.make_context_current(main_window) # render visual feedback from loaded plugins glfw.poll_events() if window_should_update() and gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["pupil_detection_enabled"] = g_pool.pupil_detection_enabled session_settings["audio_mode"] = audio.get_audio_mode() if not hide_ui: glfw.restore_window(main_window) # need to do this for windows os session_settings["window_position"] = glfw.get_window_pos(main_window) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.destroy_window(main_window) glfw.terminate() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
#glfw.ext.set_icons([(icon_data, icon_width, icon_height)]) glfw.set_window_title(w, "pyglfw test") #glfw.disable(w, glfw.AUTO_POLL_EVENTS) #glfw.enable(w, glfw.KEY_REPEAT) center_x = int(vm[0][0] / 2 - glfw.get_window_size(w)[0] / 2) center_y = int(vm[0][1] / 2 - glfw.get_window_size(w)[1] / 2) print("new window position: {!s}, {!s}".format(center_x, center_y)) glfw.set_window_pos(w, center_x, center_y) glfw.set_window_size_callback(w, on_resize) glfw.set_window_close_callback(w, on_close) glfw.set_window_refresh_callback(w, on_refresh) glfw.set_key_callback(w, on_key) glfw.set_char_callback(w, on_char) glfw.set_mouse_button_callback(w, on_button) glfw.set_cursor_pos_callback(w, on_pos) glfw.set_scroll_callback(w, on_scroll) while not glfw.window_should_close(w): glfw.poll_events() if glfw.get_key(w, glfw.KEY_E) == glfw.PRESS: break glClear(GL_COLOR_BUFFER_BIT) glfw.swap_buffers(w) glfw.close_window(w) glfw.terminate()
def demo(): global quit quit = False # Callback functions def on_resize(window, w, h): h = max(h, 1) w = max(w, 1) hdpi_factor = (glfw.get_framebuffer_size(window)[0] / glfw.get_window_size(window)[0]) w, h = w * hdpi_factor, h * hdpi_factor gui.update_window(w, h) active_window = glfw.get_current_context() glfw.make_context_current(active_window) # norm_size = normalize((w,h),glfw.get_window_size(window)) # fb_size = denormalize(norm_size,glfw.get_framebuffer_size(window)) adjust_gl_view(w, h, window) glfw.make_context_current(active_window) def on_iconify(window, iconfied): pass def on_key(window, key, scancode, action, mods): gui.update_key(key, scancode, action, mods) if action == glfw.PRESS: if key == glfw.KEY_ESCAPE: on_close(window) if mods == glfw.MOD_SUPER: if key == 67: # copy value to system clipboard # ideally copy what is in our text input area test_val = "copied text input" glfw.set_clipboard_string(window, test_val) print("set clipboard to: %s" % (test_val)) if key == 86: # copy from system clipboard clipboard = glfw.get_clipboard_string(window) print("pasting from clipboard: %s" % (clipboard)) def on_char(window, char): gui.update_char(char) def on_button(window, button, action, mods): gui.update_button(button, action, mods) # pos = normalize(pos,glfw.get_window_size(window)) # pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels def on_pos(window, x, y): hdpi_factor = float( glfw.get_framebuffer_size(window)[0] / glfw.get_window_size(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor gui.update_mouse(x, y) def on_scroll(window, x, y): gui.update_scroll(x, y) def on_close(window): global quit quit = True logger.info("Process closing from window") # get glfw started glfw.init() window = glfw.create_window(width, height, "pyglui demo", None, None) if not window: exit() glfw.set_window_pos(window, 0, 0) # Register callbacks for the window glfw.set_window_size_callback(window, on_resize) glfw.set_window_close_callback(window, on_close) glfw.set_window_iconify_callback(window, on_iconify) glfw.set_key_callback(window, on_key) glfw.set_char_callback(window, on_char) glfw.set_mouse_button_callback(window, on_button) glfw.set_cursor_pos_callback(window, on_pos) glfw.set_scroll_callback(window, on_scroll) # test out new paste function glfw.make_context_current(window) init() basic_gl_setup() print(glGetString(GL_VERSION)) class Temp(object): """Temp class to make objects""" def __init__(self): pass foo = Temp() foo.bar = 34 foo.sel = "mi" foo.selection = ["€", "mi", u"re"] foo.mytext = "some text" foo.T = True foo.L = False def set_text_val(val): foo.mytext = val # print 'setting to :',val def pr(): print("pyglui version: %s" % (ui.__version__)) gui = ui.UI() gui.scale = 1.0 thumbbar = ui.Scrolling_Menu("ThumbBar", pos=(-80, 0), size=(0, 0), header_pos="hidden") menubar = ui.Scrolling_Menu("MenueBar", pos=(-500, 0), size=(-90, 0), header_pos="left") gui.append(menubar) gui.append(thumbbar) T = ui.Growing_Menu("T menu", header_pos="headline") menubar.append(T) L = ui.Growing_Menu("L menu", header_pos="headline") menubar.append(L) M = ui.Growing_Menu("M menu", header_pos="headline") menubar.append(M) def toggle_menu(collapsed, menu): menubar.collapsed = collapsed for m in menubar.elements: m.collapsed = True menu.collapsed = collapsed thumbbar.append( ui.Thumb( "collapsed", T, label="T", on_val=False, off_val=True, setter=lambda x: toggle_menu(x, T), )) thumbbar.append( ui.Thumb( "collapsed", L, label="L", on_val=False, off_val=True, setter=lambda x: toggle_menu(x, L), )) thumbbar.append( ui.Thumb( "collapsed", M, label="M", on_val=False, off_val=True, setter=lambda x: toggle_menu(x, M), )) # thumbbar.elements[-1].order = -10.0 print("order" + str(thumbbar.elements[-1].order)) T.append(ui.Button("T test", pr)) T.append( ui.Info_Text( "T best finerfpiwnesdco'n wfo;ineqrfo;inwefo'qefr voijeqfr'p9qefrp'i 'iqefr'ijqfr eqrfiqerfn'ioer" )) L.append(ui.Button("L test", pr)) L.append(ui.Button("L best", pr)) M.append(ui.Button("M test", pr)) M.append(ui.Button("M best", pr)) MM = ui.Growing_Menu("MM menu", pos=(0, 0), size=(0, 400)) M.append(MM) for x in range(20): MM.append(ui.Button("M test%s" % x, pr)) M.append(ui.Button("M best", pr)) M.append(ui.Button("M best", pr)) M.append(ui.Button("M best", pr)) M.append(ui.Button("M best", pr)) M.append(ui.Button("M best", pr)) M.append(ui.Button("M best", pr)) M.append(ui.Button("M best", pr)) # label = 'Ï' # label = 'R' # gui.append( import os import psutil pid = os.getpid() ps = psutil.Process(pid) ts = time.time() from pyglui import graph print(graph.__version__) cpu_g = graph.Line_Graph() cpu_g.pos = (50, 100) cpu_g.update_fn = ps.cpu_percent cpu_g.update_rate = 5 cpu_g.label = "CPU %0.1f" fps_g = graph.Line_Graph() fps_g.pos = (50, 100) fps_g.update_rate = 5 fps_g.label = "%0.0f FPS" fps_g.color[:] = 0.1, 0.1, 0.8, 0.9 on_resize(window, *glfw.get_window_size(window)) while not quit: gui.update() # print(T.collapsed,L.collapsed,M.collapsed) # T.collapsed = True # glfw.make_context_current(window) glfw.swap_buffers(window) glfw.poll_events() # adjust_gl_view(1280,720,window) glClearColor(0.3, 0.4, 0.1, 1) glClear(GL_COLOR_BUFFER_BIT) gui.terminate() glfw.terminate() logger.debug("Process done")
key = key_map[k] else: # not a key we handle in the key callback return if mods & glfw.MOD_CONTROL: event_pipe.append(('key_combo', 'ctrl', key)) else: event_pipe.append(('key_press', key)) def scroll_callback(_win, dx, dy): target_scroll[0] += dx * line_height * 2 target_scroll[1] += dy * line_height * 2 scroll(target_scroll[0] + dx * 20, target_scroll[1] + dy * 20) glfw.set_scroll_callback(window, scroll_callback) glfw.set_char_callback(window, char_callback) glfw.set_key_callback(window, key_callback) globs = {} globs['plt'] = importlib.import_module('matplotlib.pyplot') # wrap plt.plot _orig_plot = globs['plt'].plot @wraps(_orig_plot) def wrapped_plot(*args, **kwargs): wrapped_plot.was_called = True return _orig_plot(*args, **kwargs) last_frame = 0 with skia_surface(window) as surface:
def glfw_assign_event_callbacks(window): glfw.poll_events() glfw.set_key_callback(window, glfw_key_callback) glfw.set_char_callback(window, glfw_Char_Callback) glfw.set_mouse_button_callback(window, glfw_mouse_button_callback)
def __init__( self, g_pool, window_size=window_size_default, window_position=None, gui_scale=1.0, ui_config={}, ): super().__init__(g_pool) self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128 glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if g_pool.hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window main_window = glfw.create_window(*window_size, "Pupil Service", None, None) window_position_manager = gl_utils.WindowPositionManager() window_position = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=window_position, ) glfw.set_window_pos(main_window, *window_position) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(0, 0), size=(0, 0), header_pos="headline") g_pool.gui.append(g_pool.menubar) # Callback functions def on_resize(window, w, h): # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) self.window_size = w, h self.content_scale = gl_utils.get_content_scale(window) g_pool.gui.scale = self.content_scale g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() # Needed, to update the window buffer while resizing self.update_ui() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def set_window_size(): # Get default window size f_width, f_height = window_size_default # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.set_window_should_close(main_window, True) self.notify_all({"subject": "clear_settings_process.should_start"}) self.notify_all({ "subject": "service_process.should_start", "delay": 2.0 }) g_pool.menubar.append(ui.Button("Reset window size", set_window_size)) pupil_remote_addr = "{}:{}".format( socket.gethostbyname(socket.gethostname()), g_pool.preferred_remote_port) g_pool.menubar.append( ui.Text_Input( "pupil_remote_addr", getter=lambda: pupil_remote_addr, setter=lambda x: None, label="Pupil Remote address", )) g_pool.menubar.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: self.start_stop_eye(0, alive), getter=lambda: g_pool.eye_procs_alive[0].value, )) g_pool.menubar.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: self.start_stop_eye(1, alive), getter=lambda: g_pool.eye_procs_alive[1].value, )) g_pool.menubar.append( ui.Info_Text("Service Version: {}".format(g_pool.version))) g_pool.menubar.append( ui.Button("Restart with default settings", reset_restart)) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) g_pool.gui.configuration = ui_config gl_utils.basic_gl_setup() on_resize(g_pool.main_window, *glfw.get_framebuffer_size(main_window))
def main(): if not glfw.init(): return glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 4) glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 1) glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, True) glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) glfw.window_hint(glfw.DOUBLEBUFFER, 0) glfw.window_hint(glfw.SAMPLES, 16) width = 1920 height = 1080 aspect = 1.0 * width / height window = glfw.create_window(width, height, "Mandelbrot", None, None) if not window: glfw.terminate() return glfw.make_context_current(window) # print(f"opengl version: {gl.glGetString(gl.GL_VERSION).decode('ascii')}") # print(f"opengl vendor: {gl.glGetString(gl.GL_VENDOR).decode('ascii')}") # print(f"opengl renderer: {gl.glGetString(gl.GL_RENDERER).decode('ascii')}") vertex_shader = make_shader(gl.GL_VERTEX_SHADER, vertex_shader_src) fragment_shader = make_shader(gl.GL_FRAGMENT_SHADER, fragment_shader_src) program = make_program([vertex_shader, fragment_shader]) vert_values = numpy.array([ -1, -1 * aspect, 0, 1, -1 * aspect, 0, -1, 1 * aspect, 0, -1, 1 * aspect, 0, 1, -1 * aspect, 0, 1, 1 * aspect, 0, ], dtype='float64') # creating vertex array vert_array = gl.glGenVertexArrays(1) gl.glBindVertexArray(vert_array) vert_buffer = gl.glGenBuffers(1) gl.glBindBuffer(gl.GL_ARRAY_BUFFER, vert_buffer) gl.glBufferData(gl.GL_ARRAY_BUFFER, vert_values, gl.GL_STATIC_DRAW) gl.glClearColor(0, 0, 0, 0) gl.glClear(gl.GL_COLOR_BUFFER_BIT) gl.glUseProgram(program) # setup coordinate buffer gl.glEnableVertexAttribArray(0) gl.glBindBuffer(gl.GL_ARRAY_BUFFER, vert_buffer) gl.glVertexAttribPointer(0, 3, gl.GL_DOUBLE, gl.GL_FALSE, 0, None) # setup uniforms for fragment shader transform_loc = gl.glGetUniformLocation(program, 'transform') max_iters_loc = gl.glGetUniformLocation(program, 'max_iters') # setup color buffer # gl.glEnableVertexAttribArray(1) # gl.glBindBuffer(gl.GL_ARRAY_BUFFER, color_buffer) # gl.glVertexAttribPointer(1, 3, gl.GL_FLOAT, gl.GL_FALSE, 0, None) state = { 'zoom': 1, 'pos_x': -0.7600189058857209, 'pos_y': 0.0799516080512771, 'max_iters': 100, } def char_callback(window, char): ch = chr(char) change = False if ch == '-': state['zoom'] *= 1.1 state['zoom'] = min(10, state['zoom']) change = True elif ch in ('+', '='): state['zoom'] *= 0.9 change = True elif ch == ']': state['max_iters'] *= 1.1 change = True elif ch == '[': state['max_iters'] *= 0.9 change = True if change: print('Current zoom:', state['zoom']) print('Current max_iters:', state['max_iters']) def key_callback(window, key, scancode, action, mods): change = False if action in (glfw.PRESS, glfw.REPEAT): if key == glfw.KEY_UP: state['pos_y'] += state['zoom'] * 0.02 change = True elif key == glfw.KEY_DOWN: state['pos_y'] -= state['zoom'] * 0.02 change = True elif key == glfw.KEY_RIGHT: state['pos_x'] += state['zoom'] * 0.02 change = True elif key == glfw.KEY_LEFT: state['pos_x'] -= state['zoom'] * 0.02 change = True if change: print('Current center:', state['pos_x'], state['pos_y']) glfw.set_char_callback(window, char_callback) glfw.set_key_callback(window, key_callback) time_before = glfw.get_time() print("use +/- to zoom in/out") print("use [/] to increase/decrease max_iters") print("use arrows to pan") while not glfw.window_should_close(window): zoom = state['zoom'] pos_x = state['pos_x'] pos_y = state['pos_y'] gl.glUniformMatrix3dv( transform_loc, 1, False, numpy.array( [aspect * zoom, 0, pos_x, 0, 1 * zoom, pos_y, 0, 0, 1 * zoom], dtype='float64')) gl.glUniform1i(max_iters_loc, int(state['max_iters'])) gl.glDrawArrays(gl.GL_TRIANGLES, 0, int(len(vert_values) / 3)) time = glfw.get_time() print("frame render time", time - time_before) time_before = time gl.glFlush() glfw.wait_events() glfw.terminate()
def init(): imgui_renderer = graphics_subsystem.imgui_renderer window = graphics_subsystem.window def mouseclick_callback(window, button, action, modifiers): pass def scroll_callback(window, x, y): imgui_renderer.scroll_callback(window, x, y) def resize_callback(window, w, h): imgui_renderer.resize_callback(window, w, h) def char_callback(window, char): imgui_renderer.char_callback(window, char) def keyboard_callback(window, key, scancode, action, mods): global wireframe imgui_renderer.keyboard_callback(window, key, scancode, action, mods) if key == glfw.KEY_ESCAPE: glfw.set_window_should_close(window, glfw.TRUE) return if key in key_input.bind_map.keys(): con_cmd = key_input.bind_map[key] if str.startswith(con_cmd, "+"): if action == glfw.PRESS: # todo input queue console.handle_input(con_cmd) elif action == glfw.RELEASE: # todo input queue console.handle_input(str.replace(con_cmd, "+", "-", 1)) elif action == glfw.PRESS: # todo input queue console.handle_input(con_cmd) return if console.SHOW_CONSOLE: if key == glfw.KEY_ENTER: console.ENTER_PRESSED = True return # todo input queue # inputs['key'] = key def mouse_callback(window, x, y): inputs['mouse'] = [x, y] y = -y * 0.05 x = -x * 0.05 if y > 90: y = 90 if y < -90: y = -90 if console.SHOW_CONSOLE: return client_main.viewangles[0] = x % 360 client_main.viewangles[1] = y pass glfw.set_mouse_button_callback(window, mouseclick_callback) glfw.set_cursor_pos_callback(window, mouse_callback) glfw.set_key_callback(window, keyboard_callback) glfw.set_scroll_callback(window, scroll_callback) glfw.set_window_size_callback(window, resize_callback) glfw.set_char_callback(window, char_callback)
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, hide_ui=False, debug=False, pub_socket_hwm=None, parent_application="capture", ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing ``start_eye_plugin``: Start plugins in eye process Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url, pub_socket_hwm) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw from gl_utils import GLFWErrorReporting GLFWErrorReporting.set_default() from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport # monitoring import psutil # Plug-ins from plugin import Plugin_List # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import parse_version from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError from ndsi import H264Writer from video_capture import source_classes, manager_classes from roi import Roi from background_helper import IPC_Logging_Task_Proxy from pupil_detector_plugins import available_detector_plugins, EVENT_KEY IPC_Logging_Task_Proxy.push_url = ipc_push_url def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) # NOTE: Interrupt is handled in world/service/player which are responsible for # shutting down the eye process properly signal.signal(signal.SIGINT, interrupt_handler) # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None content_scale = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.debug = debug g_pool.user_dir = user_dir g_pool.version = version g_pool.app = parent_application g_pool.eye_id = eye_id g_pool.process = f"eye{eye_id}" g_pool.timebase = timebase g_pool.camera_render_size = None g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_socket g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic def load_runtime_pupil_detection_plugins(): from plugin import import_runtime_plugins from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin plugins_path = os.path.join(g_pool.user_dir, "plugins") for plugin in import_runtime_plugins(plugins_path): if not isinstance(plugin, type): continue if not issubclass(plugin, PupilDetectorPlugin): continue if plugin is PupilDetectorPlugin: continue yield plugin available_detectors = available_detector_plugins() runtime_detectors = list(load_runtime_pupil_detection_plugins()) plugins = ( manager_classes + source_classes + available_detectors + runtime_detectors + [Roi] ) g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ f"Pupil Cam3 ID{eye_id}", f"Pupil Cam2 ID{eye_id}", f"Pupil Cam1 ID{eye_id}", ] if eye_id == 0: preferred_names += ["HD-6000"] default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": preferred_names, "frame_size": (192, 192), "frame_rate": 120, } default_plugins = [ # TODO: extend with plugins (default_capture_name, default_capture_settings), ("UVC_Manager", {}), *[(p.__name__, {}) for p in available_detectors], ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("Roi", {}), ] def consume_events_and_render_buffer(): glfw.make_context_current(main_window) clear_gl_screen() if all(c > 0 for c in g_pool.camera_render_size): glViewport(0, 0, *g_pool.camera_render_size) for p in g_pool.plugins: p.gl_display() glViewport(0, 0, *window_size) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # update screen glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal content_scale is_minimized = bool(glfw.get_window_attrib(window, glfw.ICONIFIED)) if is_minimized: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) active_window = glfw.get_current_context() glfw.make_context_current(window) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = content_scale g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.make_context_current(active_window) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None ) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for plugin in g_pool.plugins: if plugin.on_drop(paths): break # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id)) ) if parse_version(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() camera_is_physically_flipped = eye_id == 0 g_pool.iconified = False g_pool.capture = None g_pool.flip = session_settings.get("flip", camera_is_physically_flipped) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window title = "Pupil Capture - eye {}".format(eye_id) # Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions. # As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3 # cameras, we adjust the default eye window size to a 1:1 content aspect ratio. # The size of 500 was chosen s.t. the menu still fits. default_window_size = 500 + icon_bar_width, 500 width, height = session_settings.get("window_size", default_window_size) main_window = glfw.create_window(width, height, title, None, None) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Eye camera resolutions are too small to be used as default window sizes. # We use double their size instead. frame_scale_factor = 2 f_width *= frame_scale_factor f_height *= frame_scale_factor # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", ) ) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode] ) general_settings.append(g_pool.display_mode_info) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) plugins_to_load = session_settings.get("loaded_plugins", default_plugins) if overwrite_cap_settings: # Ensure that overwrite_cap_settings takes preference over source plugins # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) # Add runtime plugins to the list of plugins to load with default arguments, # if not already restored from session settings plugins_to_load_names = set(name for name, _ in plugins_to_load) for runtime_detector in runtime_detectors: runtime_name = runtime_detector.__name__ if runtime_name not in plugins_to_load_names: plugins_to_load.append((runtime_name, {})) g_pool.plugins = Plugin_List(g_pool, plugins_to_load) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add( g_pool.plugin_by_name[default_capture_name], default_capture_settings ) toggle_general_settings(True) g_pool.writer = None g_pool.rec_path = None # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_window_iconify_callback(main_window, on_iconify) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.get_framebuffer_size(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None if platform.system() == "Darwin": # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of # occluded windows, based on the swap interval value. This causes an FPS drop # and leads to problems when recording. To side-step this behaviour, the swap # interval is set to zero. # # Read more about window occlusion on macOS here: # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html glfw.swap_interval(0) # Event loop window_should_close = False while not window_should_close: if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: g_pool.rec_path = notification["rec_path"] raw_mode = notification["compression"] start_time_synced = notification["start_time_synced"] logger.info(f"Will save eye video to: {g_pool.rec_path}") video_path = os.path.join( g_pool.rec_path, "eye{}.mp4".format(eye_id) ) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, start_time_synced) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = MPEG_Writer(video_path, start_time_synced) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") else: # TODO: wrap recording logic into plugin g_pool.capture.intrinsics.save( g_pool.rec_path, custom_name=f"eye{eye_id}" ) finally: g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify( { "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, } ) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif ( subject.startswith("start_eye_plugin") and notification["target"] == g_pool.process ): try: g_pool.plugins.add( g_pool.plugin_by_name[notification["name"]], notification.get("args", {}), ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif ( subject.startswith("stop_eye_plugin") and notification["target"] == g_pool.process ): try: plugin_to_stop = g_pool.plugin_by_name[notification["name"]] except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") else: plugin_to_stop.alive = False g_pool.plugins.clean() for plugin in g_pool.plugins: plugin.on_notify(notification) event = {} for plugin in g_pool.plugins: plugin.recent_events(event) frame = event.get("frame") if frame: if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'.format( type(frame), frame_publish_format ) ) else: frame_publish_format_recent_warning = False pupil_socket.send( { "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], } ) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: try: g_pool.writer.write_video_frame(frame) except NonMonotonicTimestampError as e: logger.error( "Recorder received non-monotonic timestamp!" " Stopping the recording!" ) logger.debug(str(e)) ipc_socket.notify({"subject": "recording.should_stop"}) ipc_socket.notify( {"subject": "recording.should_stop", "remote_notify": "all"} ) for result in event.get(EVENT_KEY, ()): pupil_socket.send(result) # GL drawing if window_should_update(): cpu_graph.update() if is_window_visible(main_window): consume_events_and_render_buffer() glfw.poll_events() window_should_close = glfw.window_should_close(main_window) # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer.release() g_pool.writer = None session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() # save session persistent settings session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) if not hide_ui: glfw.restore_window(main_window) # need to do this for windows os session_settings["window_position"] = glfw.get_window_pos(main_window) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): # Store unscaled window size as the operating system will scale the # windows appropriately during launch on Windows and Linux. f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() for plugin in g_pool.plugins: plugin.alive = False g_pool.plugins.clean() glfw.destroy_window(main_window) g_pool.gui.terminate() glfw.terminate() logger.info("Process shutting down.")
def __init__(self): if not glfw.init(): raise RuntimeError("Unable to initialize glfw.") w_window, h_window = 800, 600 self.window = glfw.create_window(w_window, h_window, "Voxel Visualizer", None, None) if not self.window: glfw.terminate() raise RuntimeError("Unable to create window.") monitor = glfw.get_primary_monitor() mode = glfw.get_video_mode(monitor) w_mon, h_mon = mode.size # center window. glfw.set_window_pos(self.window, int(0.5 * w_mon - 0.5 * w_window), int(0.5 * h_mon - 0.5 * h_window)) # Make the window's context current glfw.make_context_current(self.window) glfw.set_framebuffer_size_callback(self.window, self.on_resize) # add mouse handlers. glfw.set_input_mode(self.window, glfw.STICKY_MOUSE_BUTTONS, glfw.TRUE) glfw.set_mouse_button_callback(self.window, self.on_mouse_btn) glfw.set_cursor_pos_callback(self.window, self.on_mouse_move) glfw.set_window_size_callback(self.window, self.on_resize) glfw.set_key_callback(self.window, self.keyboard_callback) glfw.set_char_callback(self.window, self.char_callback) glfw.set_scroll_callback(self.window, self.scroll_callback) self.voxel_dims = glow.ivec3(256, 256, 32) # read config file. CFG = yaml.safe_load(open("config/semantic-kitti.yaml", 'r')) color_dict = CFG["color_map"] self.label_colors = glow.GlTextureRectangle(1024, 1, internalFormat=GL_RGB, format=GL_RGB) cols = np.zeros((1024 * 3), dtype=np.uint8) for label_id, color in color_dict.items(): cols[3 * label_id + 0] = color[2] cols[3 * label_id + 1] = color[1] cols[3 * label_id + 2] = color[0] self.label_colors.assign(cols) self.initializeGL() # initialize imgui imgui.create_context() self.impl = GlfwRenderer(self.window, attach_callbacks=False) self.on_resize(self.window, w_window, h_window) self.data = [] self.isDrag = False self.buttonPressed = None self.cam = Camera() self.cam.lookAt(25.0, 25.0, 25.0, 0.0, 0.0, 0.0) self.currentTimestep = 0 self.sliderValue = 0 self.showLabels = True