def show_no_rec_window(): from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x] if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '%s'" % new_rec_dir) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window, True) else: logger.error("'%s' is not a valid pupil recording" % new_rec_dir) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if session_settings.get("version", VersionFormat('0.0')) < get_version(version_file): logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() w, h = session_settings.get('window_size', (1280, 720)) window_pos = session_settings.get('window_position', (0, 0)) glfwWindowHint(GLFW_RESIZABLE, 0) window = glfwCreateWindow(w, h, 'Pupil Player') glfwWindowHint(GLFW_RESIZABLE, 1) glfwMakeContextCurrent(window) glfwSetWindowPos(window, window_pos[0], window_pos[1]) glfwSetDropCallback(window, on_drop) adjust_gl_view(w, h) glfont = fontstash.Context() glfont.add_font('roboto', get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) basic_gl_setup() glClearColor(0.5, .5, 0.5, 0.0) text = 'Drop a recording directory onto this window.' tip = '(Tip: You can drop a recording directory onto the app icon.)' # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." while not glfwWindowShouldClose(window): clear_gl_screen() glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.)) glfont.set_size(w / 25.) glfont.draw_text(w / 2, .3 * h, text) glfont.set_size(w / 30.) glfont.draw_text(w / 2, .4 * h, tip) glfont.set_blur(0.96) glfont.set_color_float((1., 1., 1., 1.)) glfont.set_size(w / 25.) glfont.draw_text(w / 2, .3 * h, text) glfont.set_size(w / 30.) glfont.draw_text(w / 2, .4 * h, tip) glfwSwapBuffers(window) glfwPollEvents() session_settings['window_position'] = glfwGetWindowPos(window) session_settings['version'] = get_version(version_file) session_settings.close() del glfont glfwDestroyWindow(window)
def service(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, cap_src): """Maps pupil to gaze data, can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``start_plugin``: Starts given plugin with the given arguments ``eye_process.started``: Sets the detection method eye process ``service_process.should_stop``: Stops the service process Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``service_process.started`` ``service_process.stopped`` ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the service process each process also loads the other imports. # This is not harmful but unnecessary. #general imports import numpy as np import logging import zmq import zmq_tools #zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('pupil', )) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) poller = zmq.Poller() poller.register(pupil_sub.socket) poller.register(notify_sub.socket) #log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from version_utils import VersionFormat import audio from uvc import get_time_monotonic #trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from pupil_sync import Pupil_Sync from pupil_remote import Pupil_Remote logger.info('Application Version: %s' % version) logger.info('System Info: %s' % get_system_info()) #g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'service' g_pool.user_dir = user_dir g_pool.version = version g_pool.get_now = get_time_monotonic g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive g_pool.timebase = timebase def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [Pupil_Remote, Pupil_Sync] + runtime_plugins plugin_by_index = runtime_plugins + calibration_plugins + gaze_mapping_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Dummy_Gaze_Mapper', {}), ('HMD_Calibration', {}), ('Pupil_Remote', {})] tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_service')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '2d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start', 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject == 'service_process.should_stop': g_pool.service_should_run = False elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': service.__doc__ }) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.3) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.0) ipc_pub.notify({'subject': 'service_process.started'}) logger.warning('Process started.') g_pool.service_should_run = True # Event loop while g_pool.service_should_run: socks = dict(poller.poll()) if pupil_sub.socket in socks: t, p = pupil_sub.recv() new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze', g) #simulate the update loop. events = {} events['gaze_positions'] = new_gaze_data events['pupil_positions'] = [ p, ] for plugin in g_pool.plugins: plugin.update(frame=None, events=events) if notify_sub.socket in socks: t, n = notify_sub.recv() handle_notifications(n) for plugin in g_pool.plugins: plugin.on_notify(n) #check if a plugin need to be destroyed g_pool.plugins.clean() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'service_process.stopped'}) #shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n)
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging # networking import zmq import zmq_tools from time import sleep # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: import glfw import gl_utils from OpenGL.GL import glClearColor from version_utils import VersionFormat from file_methods import Persistent_Dict from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path from player_methods import is_pupil_rec_dir, update_recording_to_recent def on_drop(window, count, paths): nonlocal rec_dir rec_dir = paths[0].decode('utf-8') if rec_dir: if not is_pupil_rec_dir(rec_dir): rec_dir = None # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() w, h = session_settings.get('window_size', (1280, 720)) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0) window = glfw.glfwCreateWindow(w, h, 'Pupil Player') glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1) glfw.glfwMakeContextCurrent(window) glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1]) glfw.glfwSetDropCallback(window, on_drop) glfont = fontstash.Context() glfont.add_font('roboto', get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) gl_utils.basic_gl_setup() glClearColor(0.5, .5, 0.5, 0.0) text = 'Drop a recording directory onto this window.' tip = '(Tip: You can drop a recording directory onto the app icon.)' # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." while not glfw.glfwWindowShouldClose(window): fb_size = glfw.glfwGetFramebufferSize(window) hdpi_factor = float(fb_size[0] / glfw.glfwGetWindowSize(window)[0]) gl_utils.adjust_gl_view(*fb_size) if rec_dir: if is_pupil_rec_dir(rec_dir): logger.info("Starting new session with '{}'".format(rec_dir)) text = "Updating recording format." tip = "This may take a while!" else: logger.error("'{}' is not a valid pupil recording".format(rec_dir)) tip = "Oops! That was not a valid recording." rec_dir = None gl_utils.clear_gl_screen() glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.)) glfont.set_size(w/25.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text) glfont.set_size(w/30.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip) glfont.set_blur(0.96) glfont.set_color_float((1., 1., 1., 1.)) glfont.set_size(w/25.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text) glfont.set_size(w/30.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip) glfw.glfwSwapBuffers(window) if rec_dir: update_recording_to_recent(rec_dir) glfw.glfwSetWindowShouldClose(window, True) glfw.glfwPollEvents() session_settings['window_position'] = glfw.glfwGetWindowPos(window) session_settings.close() glfw.glfwDestroyWindow(window) if rec_dir: ipc_pub.notify({"subject": "player_process.should_start", "rec_dir": rec_dir}) except: import traceback trace = traceback.format_exc() logger.error('Process player_drop crashed with trace:\n{}'.format(trace)) finally: sleep(1.0)
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version, debug): # general imports import logging # networking import zmq import zmq_tools from time import sleep # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: import glfw from gl_utils import GLFWErrorReporting GLFWErrorReporting.set_default() import gl_utils from OpenGL.GL import glClearColor from version_utils import parse_version from file_methods import Persistent_Dict from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path import player_methods as pm from pupil_recording import ( assert_valid_recording_type, InvalidRecordingException, ) from pupil_recording.update import update_recording process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) def on_drop(window, paths): nonlocal rec_dir rec_dir = paths[0] if rec_dir: try: assert_valid_recording_type(rec_dir) except InvalidRecordingException as err: logger.error(str(err)) rec_dir = None # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if parse_version(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() w, h = session_settings.get("window_size", (1280, 720)) glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) glfw.window_hint(glfw.RESIZABLE, 0) window = glfw.create_window(w, h, "Pupil Player", None, None) glfw.window_hint(glfw.RESIZABLE, 1) glfw.make_context_current(window) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(window, window_pos[0], window_pos[1]) glfw.set_drop_callback(window, on_drop) glfont = fontstash.Context() glfont.add_font("roboto", get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) gl_utils.basic_gl_setup() glClearColor(0.5, 0.5, 0.5, 0.0) text = "Drop a recording directory onto this window." tip = "(Tip: You can drop a recording directory onto the app icon.)" # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." def display_string(string, font_size, center_y): x = w / 2 * content_scale y = center_y * content_scale glfont.set_size(font_size * content_scale) glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.0)) glfont.draw_text(x, y, string) glfont.set_blur(0.96) glfont.set_color_float((1.0, 1.0, 1.0, 1.0)) glfont.draw_text(x, y, string) while not glfw.window_should_close( window) and not process_was_interrupted: fb_size = glfw.get_framebuffer_size(window) content_scale = gl_utils.get_content_scale(window) gl_utils.adjust_gl_view(*fb_size) if rec_dir: try: assert_valid_recording_type(rec_dir) logger.info( "Starting new session with '{}'".format(rec_dir)) text = "Updating recording format." tip = "This may take a while!" except InvalidRecordingException as err: logger.error(str(err)) if err.recovery: text = err.reason tip = err.recovery else: text = "Invalid recording" tip = err.reason rec_dir = None gl_utils.clear_gl_screen() display_string(text, font_size=51, center_y=216) for idx, line in enumerate(tip.split("\n")): tip_font_size = 42 center_y = 288 + tip_font_size * idx * 1.2 display_string(line, font_size=tip_font_size, center_y=center_y) glfw.swap_buffers(window) if rec_dir: try: update_recording(rec_dir) except AssertionError as err: logger.error(str(err)) tip = "Oops! There was an error updating the recording." rec_dir = None except InvalidRecordingException as err: logger.error(str(err)) if err.recovery: text = err.reason tip = err.recovery else: text = "Invalid recording" tip = err.reason rec_dir = None else: glfw.set_window_should_close(window, True) glfw.poll_events() session_settings["window_position"] = glfw.get_window_pos(window) session_settings.close() glfw.destroy_window(window) if rec_dir: ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir }) except Exception: import traceback trace = traceback.format_exc() logger.error( "Process player_drop crashed with trace:\n{}".format(trace)) finally: sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation # from vis_scan_path import Vis_Scan_Path from seek_control import Seek_Control from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay assert VersionFormat(pyglui_version) >= VersionFormat( "1.23"), "pyglui out of date, please upgrade to newest version" runtime_plugins = import_runtime_plugins( os.path.join(user_dir, "plugins")) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, # Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, ] + runtime_plugins if platform.system() != "Windows": # Head pose tracking is currently not available on Windows from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) user_plugins.append(Offline_Head_Pose_Tracker) plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for path in paths: if pm.is_pupil_rec_dir(path): _restart_with_recording(path) return # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in g_pool.plugins) def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": rec_dir }) glfw.glfwSetWindowShouldClose(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) meta_info = pm.load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ][0] File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_pos = session_settings.get("window_position", window_position_default) window_name = "Pupil Player: {} - {}".format( meta_info["Recording Name"], os.path.split(rec_dir)[-1]) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.binocular = meta_info.get("Eye Mode", "monocular") == "binocular" g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( "min_data_confidence", 0.6) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) # populated by producers g_pool.pupil_positions = pm.Bisector() g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector()) g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info["Data Format Version"], "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu("User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline") g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), label="Interface Size", )) general_settings.append( ui.Info_Text("Player Version: {}".format(g_pool.version))) general_settings.append( ui.Info_Text("Capture Version: {}".format( meta_info["Capture Software Version"]))) general_settings.append( ui.Info_Text("Data Format Version: {}".format( meta_info["Data Format Version"]))) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." )) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", )) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey="e", label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), ("Pupil_From_Recording", {}), ("GazeFromRecording", {}), ("Audio_Playback", {}), ] g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add(g_pool.plugin_by_name[n["name"]], args=n.get("args", {})) elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) # call `on_click` callbacks until a plugin indicates # that it has consumed the event (by returning True) any( p.on_click(pos, button, action) for p in g_pool.plugins) for key, scancode, action, mods in user_input.keys: # call `on_key` callbacks until a plugin indicates # that it has consumed the event (by returning True) any( p.on_key(key, scancode, action, mods) for p in g_pool.plugins) for char_ in user_input.chars: # call `char_` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_char(char_) for p in g_pool.plugins) # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, debug, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``eye_process.started`` ``start_plugin`` ``should_stop`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def detection_enabled_getter() -> bool: return g_pool.pupil_detection_enabled def detection_enabled_setter(is_on: bool): g_pool.pupil_detection_enabled = is_on n = {"subject": "set_pupil_detection_enabled", "value": is_on} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw from gl_utils import GLFWErrorReporting GLFWErrorReporting.set_default() from version_utils import parse_version from pyglui import ui, cygl, __version__ as pyglui_version assert parse_version(pyglui_version) >= parse_version( "1.27"), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") import audio # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_choreography import ( available_calibration_choreography_plugins, CalibrationChoreographyPlugin, patch_loaded_plugins_with_choreography_plugin, ) available_choreography_plugins = available_calibration_choreography_plugins( ) from gaze_mapping import registered_gazer_classes from gaze_mapping.gazer_base import GazerBase from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from network_api import NetworkApiPlugin from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from accuracy_visualizer import Accuracy_Visualizer from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) icon_bar_width = 50 window_size = None camera_render_size = None content_scale = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.debug = debug g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins")) runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) ] user_plugins = [ Pupil_Groups, NetworkApiPlugin, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ([ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes) plugins = (system_plugins + user_plugins + runtime_plugins + available_choreography_plugins + registered_gazer_classes()) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, CalibrationChoreographyPlugin, GazerBase, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ (default_capture_name, default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), # Calibration choreography plugin is added below by calling # patch_loaded_plugins_with_choreography_plugin ("Recorder", {}), ("NetworkApiPlugin", {}), ("Fixation_Detector", {}), ("Blink_Detection", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] def consume_events_and_render_buffer(): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal content_scale if w == 0 or h == 0: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for plugin in g_pool.plugins: if plugin.on_drop(paths): break tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world")) if parse_version(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_data_confidence = 0.6 g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) g_pool.pupil_detection_enabled = session_settings.get( "pupil_detection_enabled", True) g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.set_audio_mode( session_settings.get("audio_mode", audio.get_default_audio_mode())) def handle_notifications(noti): subject = noti["subject"] if subject == "set_pupil_detection_enabled": g_pool.pupil_detection_enabled = noti["value"] elif subject == "start_plugin": try: g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_pupil_detection_enabled", "value": g_pool.pupil_detection_enabled, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) elif subject == "world_process.adapt_window_size": set_window_size() elif subject == "world_process.should_stop": glfw.set_window_should_close(main_window, True) width, height = session_settings.get("window_size", (1280 + icon_bar_width, 720)) # window and gl setup glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window main_window = glfw.create_window(width, height, "Pupil Capture - World", None, None) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.set_window_should_close(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "world_process.should_start", "delay": 2.0 }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "Audio mode", None, getter=audio.get_audio_mode, setter=audio.set_audio_mode, selection=audio.get_audio_mode_list(), )) general_settings.append( ui.Switch( "pupil_detection_enabled", label="Pupil detection", getter=detection_enabled_getter, setter=detection_enabled_setter, )) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, )) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, )) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version))) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) loaded_plugins = session_settings.get("loaded_plugins", default_plugins) # Resolve the active calibration choreography plugin loaded_plugins = patch_loaded_plugins_with_choreography_plugin( loaded_plugins, app=g_pool.app) session_settings["loaded_plugins"] = loaded_plugins # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, loaded_plugins) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add(g_pool.plugin_by_name[default_capture_name], default_capture_settings) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes g_pool.trigger_main_window_redraw = lambda: on_resize( main_window, *glfw.get_framebuffer_size(main_window)) g_pool.trigger_main_window_redraw() if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") if platform.system() == "Darwin": # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of # occluded windows, based on the swap interval value. This causes an FPS drop # and leads to problems when recording. To side-step this behaviour, the swap # interval is set to zero. # # Read more about window occlusion on macOS here: # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html glfw.swap_interval(0) # Event loop while not glfw.window_should_close( main_window) and not process_was_interrupted: # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.make_context_current(main_window) # render visual feedback from loaded plugins glfw.poll_events() if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings[ "pupil_detection_enabled"] = g_pool.pupil_detection_enabled session_settings["audio_mode"] = audio.get_audio_mode() if not hide_ui: glfw.restore_window(main_window) # need to do this for windows os session_settings["window_position"] = glfw.get_window_pos( main_window) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.destroy_window(main_window) glfw.terminate() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] # logger.setLevel(logging.DEBUG) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { 'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id, 'delay': 0.2 } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) try: # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( '1.15'), 'pyglui out of date, please upgrade to newest version' from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes, Base_Manager, Base_Source from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay ] system_plugins = [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs ] + manager_classes + source_classes plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins user_plugins += [ p for p in runtime_plugins if not isinstance(p, (Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin)) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { 'preferred_names': [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e" ], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Plugin_Manager', {}), ('System_Graphs', {})] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] for p in g_pool.plugins: p.on_drop(paths) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add( g_pool.plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'stop_plugin': for p in g_pool.plugins: if p.class_name == n['name']: p.alive = False g_pool.plugins.clean() elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) width, height = session_settings.get('window_size', (1280 + icon_bar_width, 720)) # window and gl setup glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = camera_render_size[0] + \ int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \ glfw.glfwGetFramebufferSize(main_window)[1] logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'world_process.should_start', 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.6, .8, 1., 1.2, 1.4], label='Interface size')) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings.append(ui.Button('Reset window size', set_window_size)) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['disabled', '2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) general_settings.append( ui.Info_Text('Capture Version: {}'.format(g_pool.version))) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events['dt'] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # send new events to ipc: del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'depth_frame' in events: del events['depth_frame'] if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = '' g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings[ 'detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings['window_size'] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error('Process Capture crashed with trace:\n{}'.format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) sleep(1.0)
def world(g_pool, cap_src, cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {})] # Callback functions def on_resize(window, w, h): if not g_pool.iconified: g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': cap_size, 'frame_rate': 24} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get( 'pupil_confidence_threshold', .6) g_pool.active_calibration_plugin = None def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() #window and gl setup glfwInit() width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize(main_window, frame.width, frame.height))) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) general_settings.append( ui.Slider('pupil_confidence_threshold', g_pool, step=.01, min=0., max=1., label='Minimum pupil confidence')) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append( ui.Hot_Key("quit", setter=on_close, getter=lambda: True, label="X", hotkey=GLFW_KEY_ESCAPE)) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetWindowIconifyCallback(main_window, on_iconify) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Stopping.") break except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfwSwapBuffers(main_window) glfwPollEvents() glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings[ 'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfwDestroyWindow(main_window) glfwTerminate() cap.close() logger.debug("Process done")
def main(): # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize( window)[0] w, h = w * hdpi_factor, h * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) glfwMakeContextCurrent(active_window) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * y_scroll_factor) def on_close(window): glfwSetWindowShouldClose(main_window, True) logger.debug('Process closing from window') try: rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: if getattr(sys, 'frozen', False): logger.warning( "You did not supply a data directory when you called this script! \ \nPlease drag a Pupil recoding directory onto the launch icon." ) else: logger.warning( "You did not supply a data directory when you called this script! \ \nPlease supply a Pupil recoding directory as first arg when calling Pupil Player." ) return if not is_pupil_rec_dir(rec_dir): logger.error( "You did not supply a dir with the required files inside.") return # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) #backwards compatibility fn. patch_meta_info(rec_dir) #parse info.csv file meta_info_path = rec_dir + "/info.csv" with open(meta_info_path) as info: meta_info = dict( ((line.strip().split('\t')) for line in info.readlines())) rec_version = read_rec_version(meta_info) if rec_version < VersionFormat('0.4'): video_path = rec_dir + "/world.avi" timestamps_path = rec_dir + "/timestamps.npy" else: video_path = rec_dir + "/world.mkv" timestamps_path = rec_dir + "/world_timestamps.npy" gaze_positions_path = rec_dir + "/gaze_positions.npy" #load gaze information gaze_list = np.load(gaze_positions_path) timestamps = np.load(timestamps_path) #correlate data if rec_version < VersionFormat('0.4'): positions_by_frame = correlate_gaze_legacy(gaze_list, timestamps) else: positions_by_frame = correlate_gaze(gaze_list, timestamps) # Initialize capture cap = autoCreateCapture(video_path, timestamps=timestamps_path) if isinstance(cap, FakeCapture): logger.error("could not start capture.") return width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', (0, 0)) # not yet using this one. # Initialize glfw glfwInit() main_window = glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfwMakeContextCurrent(main_window) cygl.utils.init() # Register callbacks main_window glfwSetWindowSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # create container for globally scoped vars (within world) g_pool = Global_Container() g_pool.app = 'player' g_pool.version = get_version(version_file) g_pool.capture = cap g_pool.timestamps = timestamps g_pool.gaze_list = gaze_list g_pool.positions_by_frame = positions_by_frame g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.rec_version = rec_version g_pool.meta_info = meta_info def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: pass g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 2) except FileSeekError: pass g_pool.new_seek = True def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def get_scale(): return g_pool.gui.scale def open_plugin(plugin): if plugin == "Select to load": return logger.debug('Open Plugin: %s' % plugin) new_plugin = plugin(g_pool) g_pool.plugins.add(new_plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 300)) g_pool.main_menu.append(ui.Button("quit", lambda: on_close(None))) g_pool.main_menu.configuration = session_settings.get( 'main_menu_config', {}) g_pool.main_menu.append( ui.Slider('scale', setter=set_scale, getter=get_scale, step=.05, min=0.75, max=2.5, label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: %s' % g_pool.version)) g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' % rec_version)) g_pool.main_menu.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.main_menu.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize( main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label='Play', hotkey=GLFW_KEY_SPACE) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT) g_pool.backward_button = ui.Thumb('backward', getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT) g_pool.quickbar.extend( [g_pool.play_button, g_pool.forward_button, g_pool.backward_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) #we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Export_Launcher', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins + previous_plugins) for p in g_pool.plugins: if p.class_name == 'Trim_Marks': g_pool.trim_marks = p break #set the last saved window size on_resize(main_window, *glfwGetWindowSize(main_window)) glfwSetWindowPos(main_window, 0, 0) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture((height, width, 3)) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_now() - .03 cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.get_cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" while not glfwWindowShouldClose(main_window): #grab new frame if g_pool.play or g_pool.new_seek: try: new_frame = cap.get_frame() except EndofVideoFileError: #end of video logic: pause at last frame. g_pool.play = False if g_pool.new_seek: display_time = new_frame.timestamp g_pool.new_seek = False update_graph = True else: update_graph = False frame = new_frame.copy() events = {} #new positons we make a deepcopy just like the image is a copy. events['pupil_positions'] = deepcopy(positions_by_frame[frame.index]) if update_graph: #update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts != t: dt, ts = t - ts, t fps_graph.add(1. / dt) g_pool.play_button.status_text = str(frame.index) #always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() update_named_texture(g_pool.image_tex, frame.img) draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() #present frames at appropriate speed wait_time = frame.timestamp - display_time display_time = frame.timestamp try: spent_time = time() - timestamp sleep(wait_time - spent_time) except: pass timestamp = time() glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['main_menu_config'] = g_pool.main_menu.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.close() glfwDestroyWindow(main_window) glfwTerminate() logger.debug("Process done")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils # capture from video_capture import File_Source, EndofVideoFileError, FileSeekError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # monitoring import psutil # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_bar import Seek_Bar from trim_marks import Trim_Marks from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector # from manual_gaze_correction import Manual_Gaze_Correction from batch_exporter import Batch_Exporter from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration assert pyglui_version >= '1.7' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url # Initialize capture cap = File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index() + 1) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 1) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) g_pool.main_menu.append(ui.Info_Text('Open plugins')) selector_label = "Select to load" def append_selector(label, plugins): plugins.sort(key=lambda p: p.__name__) plugin_labels = [p.__name__.replace('_', ' ') for p in plugins] g_pool.main_menu.append(ui.Selector(label, selection=[selector_label] + plugins, labels=[selector_label] + plugin_labels, setter=open_plugin, getter=lambda: selector_label)) base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base] base_labels = ['Visualizer:', 'Analyser:', 'Data Source:'] launchable = user_launchable_plugins.copy() for base_class, label in zip(base_plugins, base_labels): member_plugins = [] for p in user_launchable_plugins: if issubclass(p, base_class): member_plugins.append(p) launchable.remove(p) append_selector(label, member_plugins) # launchable only contains plugins that could not be assigned to any of the above categories append_selector('Other', launchable) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=glfw.GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=glfw.GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=glfw.GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_ndarray(frame.bgr) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() unused_elements = g_pool.gui.update() for b in unused_elements.buttons: button, action, mods = b pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) # present frames at appropriate speed cap.wait(frame) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'})
def world(g_pool, cap_src, cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from g_pool.pupil_queue Can run various plug-ins. """ # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) norm_size = normalize((w, h), glfwGetWindowSize(window)) fb_size = denormalize(norm_size, glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int, fb_size)) adjust_gl_view(w, h, window) glfwMakeContextCurrent(active_window) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconfied): if not isinstance(cap, FakeCapture): g_pool.update_textures.value = not iconfied def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key, action): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: on_close(window) def on_char(window, char): if not atb.TwEventCharGLFW(char, 1): pass def on_button(window, button, action, mods): if not atb.TwEventMouseButtonGLFW(button, action): pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(world_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): norm_pos = normalize((x, y), glfwGetWindowSize(window)) fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x), int(fb_y)): pass def on_scroll(window, x, y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) def load(var_name, default): return session_settings.get(var_name, default) def save(var_name, var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return height, width = frame.img.shape[:2] # load last calibration data try: pt_cloud = np.load(os.path.join(g_pool.user_dir, 'cal_pt_cloud.npy')) logger.debug("Using calibration found in %s" % g_pool.user_dir) map_pupil = calibrate.get_map_from_cloud(pt_cloud, (width, height)) except: logger.debug("No calibration found.") def map_pupil(vector): """ 1 to 1 mapping """ return vector # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.plugins = [] g_pool.map_pupil = map_pupil g_pool.update_textures = c_bool(1) if isinstance(cap, FakeCapture): g_pool.update_textures.value = False g_pool.capture = cap g_pool.rec_name = recorder.get_auto_name() # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1. / dt - bar.fps.value) def set_window_size(mode, data): height, width = frame.img.shape[:2] ratio = (1, .75, .5, .25)[mode] w, h = int(width * ratio), int(height * ratio) glfwSetWindowSize(world_window, w, h) data.value = mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def set_rec_dir(val): try: n_path = os.path.expanduser(val.value) logger.debug("Expanded user path.") except: n_path = val.value if not n_path: logger.warning("Please specify a path.") elif not os.path.isdir(n_path): logger.warning("This is not a valid path.") else: g_pool.rec_dir = n_path def get_rec_dir(): return create_string_buffer(g_pool.rec_dir, 512) def set_rec_name(val): if not val.value: g_pool.rec_name = recorder.get_auto_name() else: g_pool.rec_name = val.value def get_rec_name(): return create_string_buffer(g_pool.rec_name, 512) def open_calibration(selection, data): # prepare destruction of current ref_detector... and remove it for p in g_pool.plugins: if isinstance(p, calibration_routines.detector_by_index): p.alive = False g_pool.plugins = [p for p in g_pool.plugins if p.alive] new_ref_detector = calibration_routines.detector_by_index[selection]( g_pool, atb_pos=bar.next_atb_pos) g_pool.plugins.append(new_ref_detector) g_pool.plugins.sort(key=lambda p: p.order) # save the value for atb bar data.value = selection def toggle_record_video(): for p in g_pool.plugins: if isinstance(p, recorder.Recorder): p.alive = False return new_plugin = recorder.Recorder(g_pool, g_pool.rec_name, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx, bar.audio.value) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_show_calib_result(): for p in g_pool.plugins: if isinstance(p, Show_Calibration): p.alive = False return new_plugin = Show_Calibration(g_pool, frame.img.shape) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_server(): for p in g_pool.plugins: if isinstance(p, Pupil_Server): p.alive = False return new_plugin = Pupil_Server(g_pool, (10, 300)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_remote(): for p in g_pool.plugins: if isinstance(p, Pupil_Remote): p.alive = False return new_plugin = Pupil_Remote(g_pool, (10, 360), on_char) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_ar(): for p in g_pool.plugins: if isinstance(p, Marker_Detector): p.alive = False return new_plugin = Marker_Detector(g_pool, (10, 400)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def reset_timebase(): #the last frame from worldcam will be t0 g_pool.timebase.value = g_pool.capture.get_now() logger.info( "New timebase set to %s all timestamps will count from here now." % g_pool.timebase.value) atb.init() # add main controls ATB bar bar = atb.Bar(name="World", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100, valueswidth=150, text='light', position=(10, 10), refresh=.3, size=(300, 200)) bar.next_atb_pos = (10, 220) bar.fps = c_float(0.0) bar.timestamp = time() bar.calibration_type = c_int(load("calibration_type", 0)) bar.record_eye = c_bool(load("record_eye", 0)) bar.audio = c_int(load("audio", -1)) bar.window_size = c_int(load("window_size", 0)) window_size_enum = atb.enum("Display Size", { "Full": 0, "Medium": 1, "Half": 2, "Mini": 3 }) calibrate_type_enum = atb.enum("Calibration Method", calibration_routines.index_by_name) audio_enum = atb.enum("Audio Input", dict(Audio_Input_List())) bar.version = create_string_buffer(g_pool.version, 512) bar.add_var( "fps", bar.fps, step=1., readonly=True, help= "Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate." ) bar.add_var( "display size", vtype=window_size_enum, setter=set_window_size, getter=get_from_data, data=bar.window_size, help="Resize the world window. This has no effect on the actual image." ) bar.add_var("calibration method", setter=open_calibration, getter=get_from_data, data=bar.calibration_type, vtype=calibrate_type_enum, group="Calibration", help="Please choose your desired calibration method.") bar.add_button("show calibration result", toggle_show_calib_result, group="Calibration", help="Click to show calibration result.") bar.add_var("rec dir", create_string_buffer(512), getter=get_rec_dir, setter=set_rec_dir, group="Recording", help="Specify the recording path") bar.add_var("session name", create_string_buffer(512), getter=get_rec_name, setter=set_rec_name, group="Recording", help="Give your recording session a custom name.") bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording") bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye") bar.add_var("record audio", bar.audio, vtype=audio_enum, group="Recording", help="Select from audio recording options.") bar.add_button( "start/stop marker tracking", toggle_ar, key="x", help="find markers in scene to map gaze onto referace surfaces") bar.add_button( "start/stop server", toggle_server, key="s", help= "the server broadcasts pupil and gaze positions locally or via network" ) bar.add_button("start/stop remote", toggle_remote, key="w", help="remote allows seding commad to pupil via network") bar.add_button( "set timebase to now", reset_timebase, help="this button allows the timestamps to count from now on.", key="t") bar.add_var( "update screen", g_pool.update_textures, help= "if you dont need to see the camera image updated, you can turn this of to reduce CPU load." ) bar.add_separator("Sep1") bar.add_var("version", bar.version, readonly=True) bar.add_var("exit", g_pool.quit) # add uvc camera controls ATB bar cap.create_atb_bar(pos=(320, 10)) # Initialize glfw glfwInit() world_window = glfwCreateWindow(width, height, "World", None, None) glfwMakeContextCurrent(world_window) # Register callbacks world_window glfwSetWindowSizeCallback(world_window, on_resize) glfwSetWindowCloseCallback(world_window, on_close) glfwSetWindowIconifyCallback(world_window, on_iconify) glfwSetKeyCallback(world_window, on_key) glfwSetCharCallback(world_window, on_char) glfwSetMouseButtonCallback(world_window, on_button) glfwSetCursorPosCallback(world_window, on_pos) glfwSetScrollCallback(world_window, on_scroll) #set the last saved window size set_window_size(bar.window_size.value, bar.window_size) on_resize(world_window, *glfwGetWindowSize(world_window)) glfwSetWindowPos(world_window, 0, 0) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img) # refresh speed settings glfwSwapInterval(0) #load calibration plugin open_calibration(bar.calibration_type.value, bar.calibration_type) #load gaze_display plugin g_pool.plugins.append(Display_Recent_Gaze(g_pool)) # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break update_fps() #a container that allows plugins to post and read events events = [] #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() if p['norm_pupil'] is None: p['norm_gaze'] = None else: p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil']) recent_pupil_positions.append(p) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, recent_pupil_positions, events) #check if a plugin need to be destroyed g_pool.plugins = [p for p in g_pool.plugins if p.alive] # render camera image glfwMakeContextCurrent(world_window) make_coord_system_norm_based() if g_pool.update_textures.value: draw_named_texture(g_pool.image_tex, frame.img) else: draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() atb.draw() glfwSwapBuffers(world_window) glfwPollEvents() # de-init all running plugins for p in g_pool.plugins: p.alive = False #reading p.alive actually runs plug-in cleanup _ = p.alive save('window_size', bar.window_size.value) save('calibration_type', bar.calibration_type.value) save('record_eye', bar.record_eye.value) save('audio', bar.audio.value) session_settings.close() cap.close() atb.terminate() glfwDestroyWindow(world_window) glfwTerminate() logger.debug("Process done")
def show_no_rec_window(): from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path global rec_dir def on_drop(window, count, paths): global rec_dir rec_dir = paths[0].decode('utf-8') # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != get_version(version_file): logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() w, h = session_settings.get('window_size', (1280, 720)) window_pos = session_settings.get('window_position', window_position_default) glfwWindowHint(GLFW_RESIZABLE, 0) window = glfwCreateWindow(w, h, 'Pupil Player') glfwWindowHint(GLFW_RESIZABLE, 1) glfwMakeContextCurrent(window) glfwSetWindowPos(window, window_pos[0], window_pos[1]) glfwSetDropCallback(window, on_drop) glfont = fontstash.Context() glfont.add_font('roboto', get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) gl_utils.basic_gl_setup() glClearColor(0.5, .5, 0.5, 0.0) text = 'Drop a recording directory onto this window.' tip = '(Tip: You can drop a recording directory onto the app icon.)' # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." while not glfwWindowShouldClose(window): fb_size = glfwGetFramebufferSize(window) hdpi_factor = float(fb_size[0] / glfwGetWindowSize(window)[0]) gl_utils.adjust_gl_view(*fb_size) if rec_dir: if is_pupil_rec_dir(rec_dir): logger.info("Starting new session with '{}'".format(rec_dir)) text = "Updating recording format." tip = "This may take a while!" else: logger.error("'{}' is not a valid pupil recording".format(rec_dir)) tip = "Oops! That was not a valid recording." rec_dir = None gl_utils.clear_gl_screen() glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.)) glfont.set_size(w/25.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text) glfont.set_size(w/30.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip) glfont.set_blur(0.96) glfont.set_color_float((1., 1., 1., 1.)) glfont.set_size(w/25.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text) glfont.set_size(w/30.*hdpi_factor) glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip) glfwSwapBuffers(window) if rec_dir: update_recording_to_recent(rec_dir) glfwSetWindowShouldClose(window, True) glfwPollEvents() session_settings['window_position'] = glfwGetWindowPos(window) session_settings['version'] = str(get_version(version_file)) session_settings.close() del glfont glfwDestroyWindow(window)
def session(rec_dir): plugin_dir = os.path.join(user_dir, 'plugins') if not os.path.isdir(plugin_dir): os.mkdir(plugin_dir) runtime_plugins = import_runtime_plugins(plugin_dir) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__) analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__) other_plugins = sorted([Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__) user_plugins = sorted(runtime_plugins, key=lambda x: x.__name__) user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) app_version = get_version(version_file) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) < get_version(version_file): logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.pupil_data = pupil_data g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index()-2) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.delayed_notifications[notification['subject']] = notification def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark) export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(export_range.start, export_range.stop)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.notifications.append(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) selector_label = "Select to load" vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins] analysis_labels = [" " + p.__name__.replace('_', ' ') for p in analysis_plugins] other_labels = [" " + p.__name__.replace('_', ' ') for p in other_plugins] user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins] plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label] + other_plugins + [selector_label] + user_plugins) labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"] + other_labels + ["User added"] + user_labels) g_pool.main_menu.append(ui.Selector('Open plugin:', selection=plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) while not glfwWindowShouldClose(main_window): # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() g_pool.gui.update() # present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfwDestroyWindow(main_window)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ is_alive = Is_Alive_Manager(is_alive_flag) with is_alive: import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id), mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'Eye' + str(eye_id) + ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'EYE' + str(eye_id) + ' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) logging.getLogger("libav").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. #general imports import numpy as np import cv2 #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere import OpenGL.GL as gl from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, make_coord_system_eye_camera_based from ui_roi import UIRoi #monitoring import psutil import math # helpers/utils from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from av_writer import JPEG_Writer, AV_Writer # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D } #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # Callback functions def on_resize(window, w, h): if not g_pool.iconified: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size + 40, g_pool.u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (640, 480), 'frame_rate': 60} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go # pipe_to_world.send('eye%s process ready'%eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below." } g_pool.u_r = UIRoi(frame.img.shape) g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get())) def on_frame_size_change(new_size): g_pool.u_r = UIRoi((new_size[1], new_size[0])) cap.on_frame_size_change = on_frame_size_change writer = None pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "eye %s" % eye_id width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfw.glfwSwapInterval(0) sphere = Sphere(20) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector( 'pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) # let detector add its GUI g_pool.pupil_detector.init_gui(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1 / 60.) def window_should_update(): return next(window_update_timer) # Event loop while not glfw.glfwWindowShouldClose(main_window): if pipe_to_world.poll(): cmd = pipe_to_world.recv() if cmd == 'Exit': break elif cmd == "Ping": pipe_to_world.send("Pong") command = None else: command, payload = cmd if command == 'Set_Detection_Mapping_Mode': if payload == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: set_detector(Detector_2D) detector_selector.read_only = False else: command = None # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") cap.seek_to_frame(0) frame = cap.get_frame() #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if 'Rec_Start' == command: record_path, raw_mode = payload logger.info("Will save eye video to: %s" % record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy" % eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = JPEG_Writer(video_path, cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = AV_Writer(video_path, cap.frame_rate) timestamps = [] elif 'Rec_Stop' == command: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() window_size = glfw.glfwGetWindowSize(main_window) if result['method'] == '3D c++': focal_length = 620. make_coord_system_eye_camera_based( (frame.width, frame.height), focal_length) sphere_result = result['sphere'] sphere_center = sphere_result['center'] sphere_radius = sphere_result['radius'] gl.glTranslate(sphere_center[0], sphere_center[1], sphere_center[2]) gl.glScale(sphere_radius, sphere_radius, sphere_radius) sphere.draw(color=RGBA(1, 1, 1, 0.2), primitive_type=gl.GL_TRIANGLE_STRIP) # switch to work in pixel space make_coord_system_pixel_based( (frame.height, frame.width, 3), g_pool.flip) if result['confidence'] > 0: if result.has_key('ellipse'): pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result[ 'confidence'] * 0.7 #scale it a little draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': g_pool.u_r.draw(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize( ) #detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = g_pool.version session_settings[ 'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings[ 'pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() logger.debug("Process done")
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, hide_ui=False, debug=False, pub_socket_hwm=None, parent_application="capture", ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing ``start_eye_plugin``: Start plugins in eye process Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url, pub_socket_hwm) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw glfw.ERROR_REPORTING = "raise" from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture import gl_utils from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport # monitoring import psutil # Plug-ins from plugin import Plugin_List # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import parse_version from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError from ndsi import H264Writer from video_capture import source_classes, manager_classes from roi import Roi from background_helper import IPC_Logging_Task_Proxy from pupil_detector_plugins import available_detector_plugins, EVENT_KEY IPC_Logging_Task_Proxy.push_url = ipc_push_url def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) # NOTE: Interrupt is handled in world/service/player which are responsible for # shutting down the eye process properly signal.signal(signal.SIGINT, interrupt_handler) # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None content_scale = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.debug = debug g_pool.user_dir = user_dir g_pool.version = version g_pool.app = parent_application g_pool.eye_id = eye_id g_pool.process = f"eye{eye_id}" g_pool.timebase = timebase g_pool.camera_render_size = None g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_socket g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic def load_runtime_pupil_detection_plugins(): from plugin import import_runtime_plugins from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin plugins_path = os.path.join(g_pool.user_dir, "plugins") for plugin in import_runtime_plugins(plugins_path): if not isinstance(plugin, type): continue if not issubclass(plugin, PupilDetectorPlugin): continue if plugin is PupilDetectorPlugin: continue yield plugin available_detectors = available_detector_plugins() runtime_detectors = list(load_runtime_pupil_detection_plugins()) plugins = ( manager_classes + source_classes + available_detectors + runtime_detectors + [Roi] ) g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ f"Pupil Cam3 ID{eye_id}", f"Pupil Cam2 ID{eye_id}", f"Pupil Cam1 ID{eye_id}", ] if eye_id == 0: preferred_names += ["HD-6000"] default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": preferred_names, "frame_size": (192, 192), "frame_rate": 120, } default_plugins = [ # TODO: extend with plugins (default_capture_name, default_capture_settings), ("UVC_Manager", {}), *[(p.__name__, {}) for p in available_detectors], ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("Roi", {}), ] def consume_events_and_render_buffer(): glfw.make_context_current(main_window) clear_gl_screen() if all(c > 0 for c in g_pool.camera_render_size): glViewport(0, 0, *g_pool.camera_render_size) for p in g_pool.plugins: p.gl_display() glViewport(0, 0, *window_size) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for button, action, mods in user_input.buttons: x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # update screen glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal content_scale is_minimized = bool(glfw.get_window_attrib(window, glfw.ICONIFIED)) if is_minimized: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) active_window = glfw.get_current_context() glfw.make_context_current(window) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = content_scale g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.make_context_current(active_window) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None ) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for plugin in g_pool.plugins: if plugin.on_drop(paths): break # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id)) ) if parse_version(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() camera_is_physically_flipped = eye_id == 0 g_pool.iconified = False g_pool.capture = None g_pool.flip = session_settings.get("flip", camera_is_physically_flipped) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window title = "Pupil Capture - eye {}".format(eye_id) # Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions. # As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3 # cameras, we adjust the default eye window size to a 1:1 content aspect ratio. # The size of 500 was chosen s.t. the menu still fits. default_window_size = 500 + icon_bar_width, 500 width, height = session_settings.get("window_size", default_window_size) main_window = glfw.create_window(width, height, title, None, None) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Eye camera resolutions are too small to be used as default window sizes. # We use double their size instead. frame_scale_factor = 2 f_width *= frame_scale_factor f_height *= frame_scale_factor # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", ) ) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode] ) general_settings.append(g_pool.display_mode_info) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) plugins_to_load = session_settings.get("loaded_plugins", default_plugins) if overwrite_cap_settings: # Ensure that overwrite_cap_settings takes preference over source plugins # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) # Add runtime plugins to the list of plugins to load with default arguments, # if not already restored from session settings plugins_to_load_names = set(name for name, _ in plugins_to_load) for runtime_detector in runtime_detectors: runtime_name = runtime_detector.__name__ if runtime_name not in plugins_to_load_names: plugins_to_load.append((runtime_name, {})) g_pool.plugins = Plugin_List(g_pool, plugins_to_load) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add( g_pool.plugin_by_name[default_capture_name], default_capture_settings ) toggle_general_settings(True) g_pool.writer = None g_pool.rec_path = None # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_window_iconify_callback(main_window, on_iconify) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.get_framebuffer_size(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None if platform.system() == "Darwin": # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of # occluded windows, based on the swap interval value. This causes an FPS drop # and leads to problems when recording. To side-step this behaviour, the swap # interval is set to zero. # # Read more about window occlusion on macOS here: # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html glfw.swap_interval(0) # Event loop window_should_close = False while not window_should_close: if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: g_pool.rec_path = notification["rec_path"] raw_mode = notification["compression"] start_time_synced = notification["start_time_synced"] logger.info(f"Will save eye video to: {g_pool.rec_path}") video_path = os.path.join( g_pool.rec_path, "eye{}.mp4".format(eye_id) ) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, start_time_synced) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = MPEG_Writer(video_path, start_time_synced) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") else: # TODO: wrap recording logic into plugin g_pool.capture.intrinsics.save( g_pool.rec_path, custom_name=f"eye{eye_id}" ) finally: g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify( { "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, } ) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif ( subject.startswith("start_eye_plugin") and notification["target"] == g_pool.process ): try: g_pool.plugins.add( g_pool.plugin_by_name[notification["name"]], notification.get("args", {}), ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif ( subject.startswith("stop_eye_plugin") and notification["target"] == g_pool.process ): try: plugin_to_stop = g_pool.plugin_by_name[notification["name"]] except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") else: plugin_to_stop.alive = False g_pool.plugins.clean() for plugin in g_pool.plugins: plugin.on_notify(notification) event = {} for plugin in g_pool.plugins: plugin.recent_events(event) frame = event.get("frame") if frame: if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'.format( type(frame), frame_publish_format ) ) else: frame_publish_format_recent_warning = False pupil_socket.send( { "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], } ) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: try: g_pool.writer.write_video_frame(frame) except NonMonotonicTimestampError as e: logger.error( "Recorder received non-monotonic timestamp!" " Stopping the recording!" ) logger.debug(str(e)) ipc_socket.notify({"subject": "recording.should_stop"}) ipc_socket.notify( {"subject": "recording.should_stop", "remote_notify": "all"} ) for result in event.get(EVENT_KEY, ()): pupil_socket.send(result) # GL drawing if window_should_update(): cpu_graph.update() if is_window_visible(main_window): consume_events_and_render_buffer() glfw.poll_events() window_should_close = glfw.window_should_close(main_window) # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer.release() g_pool.writer = None session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() # save session persistent settings session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) if not hide_ui: glfw.restore_window(main_window) # need to do this for windows os session_settings["window_position"] = glfw.get_window_pos(main_window) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): # Store unscaled window size as the operating system will scale the # windows appropriately during launch on Windows and Linux. f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() for plugin in g_pool.plugins: plugin.alive = False g_pool.plugins.clean() glfw.destroy_window(main_window) g_pool.gui.terminate() glfw.terminate() logger.info("Process shutting down.")
def world(pupil_queue, timebase, launcher_pipe, eye_pipes, eyes_are_alive, user_dir, version, cap_src): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) # create formatter formatter = logging.Formatter( '%(processName)s - [%(levelname)s] %(name)s : %(message)s') # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir, 'capture.log'), mode='w') fh.setLevel(logger.level) fh.setFormatter(formatter) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #setup thread to recv log recrods from other processes. def log_loop(logging): import zmq ctx = zmq.Context() sub = ctx.socket(zmq.SUB) sub.bind('tcp://127.0.0.1:502020') sub.setsockopt(zmq.SUBSCRIBE, "") while True: record = sub.recv_pyobj() logger = logging.getLogger(record.name) logger.handle(record) import threading log_thread = threading.Thread(target=log_loop, args=(logging, )) log_thread.setDaemon(True) log_thread.start() # create logger for the context of this function logger = logging.getLogger(__name__) # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time, sleep import numpy as np #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '0.8' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio # Plug-ins from plugin import Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from pupil_server import Pupil_Server from pupil_sync import Pupil_Sync from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from pupil_remote import Pupil_Remote from log_history import Log_History logger.info('Application Version: %s' % version) logger.info('System Info: %s' % get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # g_pool.launcher_pipe = launcher_pipe g_pool.eye_pipes = eye_pipes g_pool.eyes_are_alive = eyes_are_alive #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Show_Calibration, Pupil_Remote, Pupil_Server, Pupil_Sync, Surface_Tracker, Annotation_Capture, Log_History ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {})] # Callback functions def on_resize(window, w, h): if not g_pool.iconified: g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (1280, 720), 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() launcher_pipe.send("Exit") return g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get( 'pupil_confidence_threshold', .6) g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '2d') g_pool.active_calibration_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: logger.error("Eye%s process already running." % eye_id) return launcher_pipe.send(eye_id) eye_pipes[eye_id].send( ('Set_Detection_Mapping_Mode', g_pool.detection_mapping_mode)) if blocking: #wait for ready message from eye to sequentialize startup eye_pipes[eye_id].send('Ping') eye_pipes[eye_id].recv() logger.warning('Eye %s process started.' % eye_id) def stop_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: eye_pipes[eye_id].send('Exit') if blocking: while eyes_are_alive[eye_id].value: sleep(.1) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): if new_mode == '2d': for p in g_pool.plugins: if "Vector_Gaze_Mapper" in p.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) p.alive = False g_pool.plugins.clean() for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes): if alive.value: pipe.send(('Set_Detection_Mapping_Mode', new_mode)) g_pool.detection_mapping_mode = new_mode #window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) general_settings.append( ui.Slider('pupil_confidence_threshold', g_pool, step=.01, min=0., max=1., label='Minimum pupil confidence')) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" if session_settings.get('eye1_process_alive', False): launch_eye_process(1, blocking=True) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, blocking=False) # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings g_pool.notifications.append({'subject': 'should_stop_recording'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) glFlush() make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings[ 'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0, blocking=True) stop_eye_process(1, blocking=True) #shut down laucher launcher_pipe.send("Exit") logger.info("Process Shutting down.")
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, hide_ui=False, ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing ``start_eye_plugin``: Start plugins in eye process Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport # monitoring import psutil # Plug-ins from plugin import Plugin_List # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError from ndsi import H264Writer from video_capture import source_classes, manager_classes from roi import Roi from background_helper import IPC_Logging_Task_Proxy from pupil_detector_plugins import available_detector_plugins from pupil_detector_plugins.manager import PupilDetectorManager IPC_Logging_Task_Proxy.push_url = ipc_push_url def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) # NOTE: Interrupt is handled in world/service/player which are responsible for # shutting down the eye process properly signal.signal(signal.SIGINT, interrupt_handler) # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.eye_id = eye_id g_pool.process = f"eye{eye_id}" g_pool.timebase = timebase g_pool.camera_render_size = None g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic default_detector_cls, available_detectors = available_detector_plugins( ) plugins = (manager_classes + source_classes + available_detectors + [PupilDetectorManager, Roi]) g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ f"Pupil Cam3 ID{eye_id}", f"Pupil Cam2 ID{eye_id}", f"Pupil Cam1 ID{eye_id}", ] if eye_id == 0: preferred_names += ["HD-6000"] default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": preferred_names, "frame_size": (320, 240), "frame_rate": 120, } default_plugins = [ # TODO: extend with plugins (default_capture_name, default_capture_settings), ("UVC_Manager", {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), # Detector needs to be loaded first to set `g_pool.pupil_detector` (default_detector_cls.__name__, {}), ("PupilDetectorManager", {}), ("Roi", {}), ] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for plugin in g_pool.plugins: if plugin.on_drop(paths): break # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.flip = session_settings.get("flip", False) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() if hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get("window_size", (640 + icon_bar_width, 480)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray( np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2], label="Interface Size", )) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width *= 2 f_height *= 2 f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", )) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) toggle_general_settings(False) plugins_to_load = session_settings.get("loaded_plugins", default_plugins) if overwrite_cap_settings: # Ensure that overwrite_cap_settings takes preference over source plugins # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) g_pool.plugins = Plugin_List(g_pool, plugins_to_load) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add(g_pool.plugin_by_name[default_capture_name], default_capture_settings) g_pool.writer = None # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: record_path = notification["rec_path"] raw_mode = notification["compression"] start_time_synced = notification["start_time_synced"] logger.info( "Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, start_time_synced) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = MPEG_Writer( video_path, start_time_synced) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify({ "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, }) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif (subject.startswith("start_eye_plugin") and notification["target"] == g_pool.process): try: g_pool.plugins.add( g_pool.plugin_by_name[notification["name"]], notification.get("args", {}), ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") for plugin in g_pool.plugins: plugin.on_notify(notification) event = {} for plugin in g_pool.plugins: plugin.recent_events(event) frame = event.get("frame") if frame: if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'. format(type(frame), frame_publish_format)) else: frame_publish_format_recent_warning = False pupil_socket.send({ "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: try: g_pool.writer.write_video_frame(frame) except NonMonotonicTimestampError as e: logger.error( "Recorder received non-monotonic timestamp!" " Stopping the recording!") logger.debug(str(e)) ipc_socket.notify({"subject": "recording.should_stop"}) ipc_socket.notify({ "subject": "recording.should_stop", "remote_notify": "all" }) result = event.get("pupil_detection_result", None) if result is not None: pupil_socket.send(result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() glViewport(0, 0, *g_pool.camera_render_size) for p in g_pool.plugins: p.gl_display() glViewport(0, 0, *window_size) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString( main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer.release() g_pool.writer = None session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() # save session persistent settings session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) if not hide_ui: glfw.glfwRestoreWindow( main_window) # need to do this for windows os session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() for plugin in g_pool.plugins: plugin.alive = False g_pool.plugins.clean() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging # networking import zmq import zmq_tools from time import sleep # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: import glfw import gl_utils from OpenGL.GL import glClearColor from version_utils import VersionFormat from file_methods import Persistent_Dict from pyglui.pyfontstash import fontstash from pyglui.ui import get_roboto_font_path import player_methods as pm from pupil_recording import ( assert_valid_recording_type, InvalidRecordingException, ) from pupil_recording.update import update_recording def on_drop(window, count, paths): nonlocal rec_dir rec_dir = paths[0].decode("utf-8") if rec_dir: try: assert_valid_recording_type(rec_dir) except InvalidRecordingException as err: logger.error(str(err)) rec_dir = None # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() w, h = session_settings.get("window_size", (1280, 720)) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwInit() glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0) window = glfw.glfwCreateWindow(w, h, "Pupil Player") glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1) glfw.glfwMakeContextCurrent(window) glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1]) glfw.glfwSetDropCallback(window, on_drop) glfont = fontstash.Context() glfont.add_font("roboto", get_roboto_font_path()) glfont.set_align_string(v_align="center", h_align="middle") glfont.set_color_float((0.2, 0.2, 0.2, 0.9)) gl_utils.basic_gl_setup() glClearColor(0.5, 0.5, 0.5, 0.0) text = "Drop a recording directory onto this window." tip = "(Tip: You can drop a recording directory onto the app icon.)" # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." def display_string(string, font_size, center_y): x = w / 2 * hdpi_factor y = center_y * hdpi_factor glfont.set_size(font_size * hdpi_factor) glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.0)) glfont.draw_text(x, y, string) glfont.set_blur(0.96) glfont.set_color_float((1.0, 1.0, 1.0, 1.0)) glfont.draw_text(x, y, string) while not glfw.glfwWindowShouldClose(window): fb_size = glfw.glfwGetFramebufferSize(window) hdpi_factor = glfw.getHDPIFactor(window) gl_utils.adjust_gl_view(*fb_size) if rec_dir: try: assert_valid_recording_type(rec_dir) logger.info( "Starting new session with '{}'".format(rec_dir)) text = "Updating recording format." tip = "This may take a while!" except InvalidRecordingException as err: logger.error(str(err)) if err.recovery: text = err.reason tip = err.recovery else: text = "Invalid recording" tip = err.reason rec_dir = None gl_utils.clear_gl_screen() display_string(text, font_size=51, center_y=216) for idx, line in enumerate(tip.split("\n")): tip_font_size = 42 center_y = 288 + tip_font_size * idx * 1.2 display_string(line, font_size=tip_font_size, center_y=center_y) glfw.glfwSwapBuffers(window) if rec_dir: try: update_recording(rec_dir) except AssertionError as err: logger.error(str(err)) tip = "Oops! There was an error updating the recording." rec_dir = None except InvalidRecordingException as err: logger.error(str(err)) if err.recovery: text = err.reason tip = err.recovery else: text = "Invalid recording" tip = err.reason rec_dir = None else: glfw.glfwSetWindowShouldClose(window, True) glfw.glfwPollEvents() session_settings["window_position"] = glfw.glfwGetWindowPos(window) session_settings.close() glfw.glfwDestroyWindow(window) if rec_dir: ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir }) except: import traceback trace = traceback.format_exc() logger.error( "Process player_drop crashed with trace:\n{}".format(trace)) finally: sleep(1.0)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, cap_src): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time, sleep import numpy as np import logging import zmq import zmq_tools #zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('pupil', )) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) #log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush, is_window_visible #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '1.0' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio from uvc import get_time_monotonic #trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from fixation_detector import Fixation_Detector_3D from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher logger.info('Application Version: %s' % version) logger.info('System Info: %s' % get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) #g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Pupil_Groups, Frame_Publisher, Show_Calibration, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector_3D ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Fixation_Detector_3D', {})] # Callback functions def on_resize(window, w, h): if is_window_visible(window): g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (1280, 720), 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings g_pool.iconified = False g_pool.capture = cap g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.%s' % eye_id, 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) #window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', cap.frame_size) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) selector_label = "Select to load" labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings ipc_pub.notify({'subject': 'recording.should_stop'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() recent_pupil_data = [] recent_gaze_data = [] new_notifications = [] while pupil_sub.new_data: t, p = pupil_sub.recv() pupil_graphs[p['id']].add(p['confidence']) recent_pupil_data.append(p) new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze', g) recent_gaze_data += new_gaze_data while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) events['pupil_positions'] = recent_pupil_data events['gaze_positions'] = recent_gaze_data # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() #send new events to ipc: del events['pupil_positions'] #already on the wire del events['gaze_positions'] #send earlier in this loop del events['dt'] #no need to send this for topic, data in events.iteritems(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) # render camera image glfw.glfwMakeContextCurrent(main_window) if is_window_visible(main_window): g_pool.image_tex.update_from_frame(frame) glFlush() make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins if is_window_visible(main_window): for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil0_graph.draw() pupil1_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) #shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n) zmq_ctx.destroy()
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D } # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None camera_render_size = None # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.process = 'eye{}'.format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = normalize(pos, camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size + 40, g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0] g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, g_pool.capture.frame_size) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] g_pool.capture_manager.on_drop(paths) g_pool.capture.on_drop(paths) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id))) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below." } capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager', {})) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__: c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name]( g_pool, **manager_settings) if eye_id == 0: cap_src = [ "Pupil Cam1 ID0", "HD-6000", "Integrated Camera", "HD USB Camera", "USB 2.0 Camera" ] else: cap_src = ["Pupil Cam1 ID1", "HD-6000", "Integrated Camera"] # Initialize capture default_settings = ('UVC_Source', { 'preferred_names': cap_src, 'frame_size': (640, 480), 'frame_rate': 90 }) capture_source_settings = overwrite_cap_settings or session_settings.get( 'capture_settings', default_settings) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__: c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings) assert g_pool.capture g_pool.u_r = UIRoi( (g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and tuple( roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_ui() def toggle_general_settings(collapsed): #this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get('window_size', g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray( np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings.append(ui.Button('Reset window size', set_window_size)) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) detector_selector = ui.Selector( 'pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) toggle_general_settings(False) g_pool.pupil_detector.init_ui() g_pool.capture.init_ui() g_pool.capture_manager.init_ui() g_pool.writer = None def replace_source(source_class_name, source_settings): g_pool.capture.deinit_ui() g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings) g_pool.capture.init_ui() if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = 'jpeg' # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject.startswith('eye_process.should_stop'): if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye'] and g_pool.capture.online: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info( "Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, g_pool.capture.frame_rate) elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate) else: g_pool.writer = AV_Writer( video_path, g_pool.capture.frame_rate) elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye{}'.format(eye_id), 'doc': eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format', 'jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' elif subject.startswith('start_eye_capture') and notification[ 'target'] == g_pool.process: replace_source(notification['name'], notification['args']) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get('frame') g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width): g_pool.pupil_detector.on_resolution_change( (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray else: raise AttributeError() except AttributeError: pass else: pupil_socket.send( 'frame.eye.%s' % eye_id, { 'width': frame.width, 'height': frame.height, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s' % eye_id, result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass glViewport(0, 0, *camera_render_size) make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame: if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline( pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if 'ellipse' in result: pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result['confidence'] * 0.7 draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) glViewport(0, 0, *window_size) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI g_pool.gui.update() glViewport(0, 0, *camera_render_size) make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) glViewport(0, 0, *window_size) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize( ) # detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings[ 'capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict( ) session_settings[ 'capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict( ) session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings[ 'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings[ 'pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_ui() g_pool.capture_manager.deinit_ui() g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version, debug): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir from OpenGL.GL import GL_COLOR_BUFFER_BIT # display import glfw from gl_utils import GLFWErrorReporting GLFWErrorReporting.set_default() # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import parse_version from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from pupil_recording import PupilRecording from csv_utils import write_key_value_file from hotkey import Hotkey # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from seek_control import Seek_Control from surface_tracker import Surface_Tracker_Offline # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import ( DisabledPupilProducer, Pupil_From_Recording, Offline_Pupil_Detection, ) from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from imu_timeline import IMUTimeline from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay from pupil_recording import ( assert_valid_recording_type, InvalidRecordingException, ) assert parse_version(pyglui_version) >= parse_version( "1.30.0"), "pyglui out of date, please upgrade to newest version" process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) runtime_plugins = import_runtime_plugins( os.path.join(user_dir, "plugins")) runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) ] system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, Offline_Fixation_Detector, Offline_Blink_Detection, Surface_Tracker_Offline, Raw_Data_Exporter, Annotation_Player, Log_History, DisabledPupilProducer, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, Offline_Head_Pose_Tracker, IMUTimeline, ] + runtime_plugins plugins = system_plugins + user_plugins def consume_events_and_render_buffer(): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for b in user_input.buttons: button, action, mods = b x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.swap_buffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal content_scale if w == 0 or h == 0: return # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) content_scale = gl_utils.get_content_scale(window) framebuffer_scale = gl_utils.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.set_window_size_limits( window, min_size, min_size, glfw.DONT_CARE, glfw.DONT_CARE, ) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, paths): for path in paths: try: assert_valid_recording_type(path) _restart_with_recording(path) return except InvalidRecordingException as err: logger.debug(str(err)) for plugin in g_pool.plugins: if plugin.on_drop(paths): break def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": rec_dir }) glfw.set_window_should_close(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) recording = PupilRecording(rec_dir) meta_info = recording.meta_info # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") icon_bar_width = 50 window_size = None content_scale = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() g_pool.app = "player" g_pool.process = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None video_path = recording.files().core().world().videos()[0].resolve() File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if parse_version(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}" glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) main_window = glfw.create_window(width, height, window_name, None, None) window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=session_settings.get("window_position", None), ) glfw.set_window_pos(main_window, window_pos[0], window_pos[1]) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT) # populated by producers g_pool.pupil_positions = pm.PupilDataBisector() g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() g_pool.eye_movements = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info.min_player_version, "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.set_window_should_close(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu("User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline") g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Info_Text( f"Minimum Player Version: {meta_info.min_player_version}")) general_settings.append( ui.Info_Text(f"Player Version: {g_pool.version}")) general_settings.append( ui.Info_Text( f"Recording Software: {meta_info.recording_software_name}")) general_settings.append( ui.Info_Text( f"Recording Software Version: {meta_info.recording_software_version}" )) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." )) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", )) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey=Hotkey.EXPORT_START_PLAYER_HOTKEY(), label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins _pupil_producer_plugins = [ # In priority order (first is default) ("Pupil_From_Recording", {}), ("Offline_Pupil_Detection", {}), ("DisabledPupilProducer", {}), ] _pupil_producer_plugins = list(reversed(_pupil_producer_plugins)) _gaze_producer_plugins = [ # In priority order (first is default) ("GazeFromRecording", {}), ("GazeFromOfflineCalibration", {}), ] _gaze_producer_plugins = list(reversed(_gaze_producer_plugins)) default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), *_pupil_producer_plugins, *_gaze_producer_plugins, ("Audio_Playback", {}), ] _plugins_to_load = session_settings.get("loaded_plugins", None) if _plugins_to_load is None: # If no plugins are available from a previous session, # then use the default plugin list _plugins_to_load = default_plugins else: # If there are plugins available from a previous session, # then prepend plugins that are required, but might have not been available before _plugins_to_load = [ *_pupil_producer_plugins, *_gaze_producer_plugins, *_plugins_to_load, ] g_pool.plugins = Plugin_List(g_pool, _plugins_to_load) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) glfw.set_drop_callback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.get_framebuffer_size(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add(g_pool.plugin_by_name[n["name"]], args=n.get("args", {})) elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) while not glfw.window_should_close( main_window) and not process_was_interrupted: # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.make_context_current(main_window) glfw.poll_events() # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.get_clipboard_string(main_window).decode() except (AttributeError, glfw.GLFWError): # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.set_clipboard_string(main_window, user_input.clipboard) for b in user_input.buttons: button, action, mods = b x, y = glfw.get_cursor_pos(main_window) pos = gl_utils.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.swap_buffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.get_window_pos(main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.get_window_size(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.destroy_window(main_window) except Exception: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
def service( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, hide_ui, ): """Maps pupil to gaze data, can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``start_plugin``: Starts given plugin with the given arguments ``eye_process.started``: Sets the detection method eye process ``service_process.should_stop``: Stops the service process Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``service_process.started`` ``service_process.stopped`` ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the service process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("pupil", )) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) poller = zmq.Poller() poller.register(pupil_sub.socket) poller.register(notify_sub.socket) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start", "eye_id": eye_id, "delay": delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {"subject": "eye_process.should_stop", "eye_id": eye_id} ipc_pub.notify(n) try: # helpers/utils from file_methods import Persistent_Dict from methods import delta_t, get_system_info from version_utils import VersionFormat import audio from uvc import get_time_monotonic # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from service_ui import Service_UI from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) # g_pool holds variables for this process they are accesible to all plugins g_pool = SimpleNamespace() g_pool.app = "service" g_pool.user_dir = user_dir g_pool.version = version g_pool.get_now = get_time_monotonic g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.timebase = timebase g_pool.preferred_remote_port = preferred_remote_port g_pool.hide_ui = hide_ui def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins")) user_launchable_plugins = [ Service_UI, Pupil_Groups, Pupil_Remote, Frame_Publisher, Blink_Detection, ] + runtime_plugins plugin_by_index = (runtime_plugins + calibration_plugins + gaze_mapping_plugins + user_launchable_plugins) name_by_index = [ pupil_datum.__name__ for pupil_datum in plugin_by_index ] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [ ("Service_UI", {}), ("Dummy_Gaze_Mapper", {}), ("HMD_Calibration", {}), ("Pupil_Remote", {}), ] g_pool.plugin_by_name = plugin_by_name tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_service")) if session_settings.get("version", VersionFormat("0.0")) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) g_pool.detection_mapping_mode = session_settings.get( "detection_mapping_mode", "2d") g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode) ipc_pub.notify({"subject": "service_process.started"}) logger.warning("Process started.") g_pool.service_should_run = True # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # NOTE: The Pupil_Remote plugin fails to load when the port is already in use # and will set this variable to false. Then we should not even start the eye # processes. Otherwise we would have to wait for their initialization before # attempting cleanup in Service. if g_pool.service_should_run: if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.3) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.0) def handle_notifications(n): subject = n["subject"] if subject == "set_detection_mapping_mode": if n["mode"] == "2d": if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(plugin_by_name["Dummy_Gaze_Mapper"]) g_pool.detection_mapping_mode = n["mode"] elif subject == "start_plugin": g_pool.plugins.add(plugin_by_name[n["name"]], args=n.get("args", {})) elif subject == "eye_process.started": n = { "subject": "set_detection_mapping_mode", "mode": g_pool.detection_mapping_mode, } ipc_pub.notify(n) elif subject == "service_process.should_stop": g_pool.service_should_run = False elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": service.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) # initiate ui update loop ipc_pub.notify({ "subject": "service_process.ui.should_update", "initial_delay": 1 / 40 }) # Event loop while g_pool.service_should_run and not process_was_interrupted: socks = dict(poller.poll()) if pupil_sub.socket in socks: topic, pupil_datum = pupil_sub.recv() new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum( pupil_datum) for gaze_datum in new_gaze_data: gaze_pub.send(gaze_datum) events = {} events["gaze"] = new_gaze_data events["pupil"] = [pupil_datum] for plugin in g_pool.plugins: plugin.recent_events(events=events) if notify_sub.socket in socks: topic, n = notify_sub.recv() handle_notifications(n) for plugin in g_pool.plugins: plugin.on_notify(n) # check if a plugin need to be destroyed g_pool.plugins.clean() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings[ "detection_mapping_mode"] = g_pool.detection_mapping_mode session_settings["audio_mode"] = audio.audio_mode session_settings.close() # de-init all running plugins for pupil_datum in g_pool.plugins: pupil_datum.alive = False g_pool.plugins.clean() except Exception: import traceback trace = traceback.format_exc() logger.error("Process Service crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "service_process.stopped"}) # shut down launcher n = {"subject": "launcher_process.should_stop"} ipc_pub.notify(n) sleep(1.0)
def __init__(self, g_pool, mode="Show markers and frames", min_marker_perimeter=40): super(Marker_Detector, self).__init__(g_pool) self.order = .2 # all markers that are detected in the most recent frame self.markers = [] #load camera intrinsics try: camera_calibration = load_object( os.path.join(self.g_pool.user_dir, 'camera_calibration')) except: self.camera_intrinsics = None else: same_name = camera_calibration[ 'camera_name'] == self.g_pool.capture.name same_resolution = camera_calibration[ 'resolution'] == self.g_pool.capture.frame_size if same_name and same_resolution: logger.info( 'Loaded camera calibration. 3D marker tracking enabled.') K = camera_calibration['camera_matrix'] dist_coefs = camera_calibration['dist_coefs'] resolution = camera_calibration['resolution'] self.camera_intrinsics = K, dist_coefs, resolution else: logger.info( 'Loaded camera calibration but camera name and/or resolution has changed. Please re-calibrate.' ) self.camera_intrinsics = None # all registered surfaces self.surface_definitions = Persistent_Dict( os.path.join(g_pool.user_dir, 'surface_definitions')) self.surfaces = [ Reference_Surface(saved_definition=d) for d in self.surface_definitions.get( 'realtime_square_marker_surfaces', []) if isinstance(d, dict) ] # edit surfaces self.edit_surfaces = [] #plugin state self.mode = mode self.running = True self.robust_detection = 1 self.aperture = 11 self.min_marker_perimeter = min_marker_perimeter self.locate_3d = False #debug vars self.draw_markers = 0 self.show_surface_idx = 0 self.img_shape = None self.menu = None self.button = None self.add_button = None
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, cap_src): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id): #logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #general imports import numpy as np import cv2 #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere import OpenGL.GL as gl from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, make_coord_system_eye_camera_based from ui_roi import UIRoi #monitoring import psutil import math # helpers/utils from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from av_writer import JPEG_Writer, AV_Writer # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D } #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.timebase = timebase # Callback functions def on_resize(window, w, h): if not g_pool.iconified: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size + 40, g_pool.u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (640, 480), 'frame_rate': 60} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below." } g_pool.u_r = UIRoi((cap.frame_size[1], cap.frame_size[0])) g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get())) def on_frame_size_change(new_size): g_pool.u_r = UIRoi((new_size[1], new_size[0])) cap.on_frame_size_change = on_frame_size_change writer = None pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "eye %s" % eye_id width, height = session_settings.get('window_size', cap.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() glfw.glfwSwapInterval(0) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector( 'pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) # let detector add its GUI g_pool.pupil_detector.init_gui(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1 / 60.) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject == 'eye_process.should_stop': if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye']: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info("Will save eye video to: %s" % record_path) timestamps_path = os.path.join( record_path, "eye%s_timestamps.npy" % eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = JPEG_Writer(video_path, cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = AV_Writer(video_path, cap.frame_rate) timestamps = [] elif subject == 'recording.stopped': if writer: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye%i' % eye_id, 'doc': eye.__doc__ }) # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") cap.seek_to_frame(0) frame = cap.get_frame() #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s' % eye_id, result) # GL drawing if window_should_update(): if not g_pool.iconified: glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() window_size = glfw.glfwGetWindowSize(main_window) make_coord_system_pixel_based( (frame.height, frame.width, 3), g_pool.flip) if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline( pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if result.has_key('ellipse'): pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result[ 'confidence'] * 0.7 #scale it a little draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize( ) #detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = g_pool.version session_settings[ 'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings[ 'pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() logger.info("Process shutting down.")
def main(): import argparse from textwrap import dedent from file_methods import Persistent_Dict def show_progess(jobs): no_jobs = len(jobs) width = 80 full = width / no_jobs string = "" for j in jobs: try: p = int(width * j.current_frame.value / float(j.frames_to_export.value * no_jobs)) except: p = 0 string += '[' + p * "|" + (full - p) * "-" + "]" sys.stdout.write("\r" + string) sys.stdout.flush() """Batch process recordings to produce visualizations Using simple_circle as the default visualizations Steps: - User Supplies: Directory that contains many recording(s) dirs or just one recordings dir - We walk the user supplied directory to get all data folders - Data is the list we feed to our multiprocessed - Error check -- do we have required files in each dir?: world.avi, gaze_positions.npy, timestamps.npy - Result: world_viz.avi within each original data folder """ parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=dedent('''\ *************************************************** Batch process recordings to produce visualizations The default visualization will use simple_circle Usage Example: python batch_exporter.py -d /path/to/folder-with-many-recordings -s ~/Pupil_Player/settings/user_settings -e ~/my_export_dir Arguments: -d : Specify a recording directory. This could have one or many recordings contained within it. We will recurse into the dir. -s : Specify path to Pupil Player user_settings file to use last used vizualization settings. -e : Specify export directory if you dont want the export saved within each recording dir. -p : Export a 120 frame preview only. ***************************************************\ ''')) parser.add_argument('-d', '--rec-dir', required=True) parser.add_argument('-s', '--settings-file', required=True) parser.add_argument('-e', '--export-to-dir', default=False) parser.add_argument('-c', '--basic-color', default='red') parser.add_argument('-p', '--preview', action='store_true') if len(sys.argv) == 1: print(parser.description) return args = parser.parse_args() # get the top level data folder from terminal argument data_dir = args.rec_dir if args.settings_file and os.path.isfile(args.settings_file): session_settings = Persistent_Dict( os.path.splitext(args.settings_file)[0]) # these are loaded based on user settings plugin_initializers = session_settings.get('loaded_plugins', []) session_settings.close() else: logger.error("Setting file not found or valid") return if args.export_to_dir: export_dir = args.export_to_dir if os.path.isdir(export_dir): logger.info("Exporting all vids to {}".format(export_dir)) else: logger.error("Exporting dir is not valid {}".format(export_dir)) return else: export_dir = None logger.info("Exporting into the recording dirs.") if args.preview: preview = True logger.info("Exporting first 120frames only") else: preview = False class Temp(object): pass recording_dirs = get_recording_dirs(data_dir) # start multiprocessing engine n_cpu = cpu_count() logger.info( "Using a maximum of {} CPUs to process visualizations in parallel...". format(n_cpu)) jobs = [] outfiles = set() for d in recording_dirs: j = Temp() logger.info("Adding new export: {}".format(d)) j.should_terminate = Value(c_bool, 0) j.frames_to_export = Value(c_int, 0) j.current_frame = Value(c_int, 0) j.data_dir = d j.user_dir = None j.start_frame = None if preview: j.end_frame = 30 else: j.end_frame = None j.plugin_initializers = plugin_initializers[:] if export_dir: # make a unique name created from rec_session and dir name rec_session, rec_dir = d.rsplit(os.path.sep, 2)[1:] out_name = rec_session + "_" + rec_dir + ".mp4" j.out_file_path = os.path.join(os.path.expanduser(export_dir), out_name) if j.out_file_path in outfiles: logger.error( "This export setting would try to save {} at least twice pleace rename dirs to prevent this." .format(j.out_file_path)) return outfiles.add(j.out_file_path) logger.info("Exporting to: {}".format(j.out_file_path)) else: j.out_file_path = None j.args = (j.should_terminate, j.frames_to_export, j.current_frame, j.data_dir, j.user_dir, j.start_frame, j.end_frame, j.plugin_initializers, j.out_file_path) jobs.append(j) todo = jobs[:] workers = [ Export_Process(target=export, args=todo.pop(0).args) for i in range(min(len(todo), n_cpu)) ] for w in workers: w.start() working = True while working: # cannot use pool as it does not allow shared memory working = False for i in range(len(workers)): if workers[i].is_alive(): working = True else: if todo: workers[i] = Export_Process(target=export, args=todo.pop(0).args) workers[i].start() working = True show_progess(jobs) time.sleep(.25) print('\n')
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ # modify the root logger for this process logger = logging.getLogger() # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye%s.log' % eye_id), mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'Eye' + str(eye_id) + ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'EYE' + str(eye_id) + ' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # create logger for the context of this function logger = logging.getLogger(__name__) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # Callback functions def on_resize(window, w, h): if not g_pool.iconified: active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == GLFW_RELEASE and u_r.active_edit_pt: u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == GLFW_PRESS: pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40, u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if u_r.active_edit_pt: pos = normalize((x, y), glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) u_r.move_vertex(u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': cap_size, 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go pipe_to_world.send('eye%s process ready' % eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below." } # g_pool.draw_pupil = session_settings.get('draw_pupil',True) u_r = UIRoi(frame.img.shape) u_r.set(session_settings.get('roi', u_r.get())) writer = None pupil_detector = Canny_Detector(g_pool) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] # Initialize glfw glfwInit() if g_pool.binocular: title = "Binocular eye %s" % eye_id else: title = 'Eye' width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl_init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfwSwapInterval(0) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize(main_window, frame.width, frame.height))) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append( ui.Hot_Key("quit", setter=on_close, getter=lambda: True, label="X", hotkey=GLFW_KEY_ESCAPE)) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # let detector add its GUI pupil_detector.init_gui(g_pool.sidebar) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetWindowIconifyCallback(main_window, on_iconify) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) #set the last saved window size on_resize(main_window, *glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1 / 60.) def window_should_update(): return next(window_update_timer) # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if pipe_to_world.poll(): command, raw_mode = pipe_to_world.recv() if command is not None: record_path = command logger.info("Will save eye video to: %s" % record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy" % eye_id) if raw_mode and hasattr(frame, 'jpeg_buffer'): video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = JPEG_Writer(video_path, cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mkv" % eye_id) writer = CV_Writer(video_path, float(cap.frame_rate), cap.frame_size) timestamps = [] else: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect( frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() # switch to work in pixel space make_coord_system_pixel_based((frame.height, frame.width, 3), g_pool.flip) if result['confidence'] > 0: if result.has_key('axes'): pts = cv2.ellipse2Poly((int( result['center'][0]), int(result['center'][1])), (int(result['axes'][0] / 2), int(result['axes'][1] / 2)), int(result['angle']), 0, 360, 15) cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5)) cygl_draw_points([result['center']], size=20, color=cygl_rgba(1., 0., 0., .5), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': u_r.draw(g_pool.gui.scale) #update screen glfwSwapBuffers(main_window) glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() pupil_detector.cleanup() g_pool.gui.terminate() glfwDestroyWindow(main_window) glfwTerminate() cap.close() #flushing queue in case world process did not exit gracefully while not g_pool.pupil_queue.empty(): g_pool.pupil_queue.get() g_pool.pupil_queue.close() logger.debug("Process done")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging import errno from glob import glob from time import time # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source, EndofVideoFileError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from batch_exporter import Batch_Exporter, Batch_Export from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs from blink_detection import Offline_Blink_Detection assert VersionFormat(pyglui_version) >= VersionFormat('1.16'), 'pyglui out of date, please upgrade to newest version' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export] user_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None # sets itself to g_pool.capture File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() g_pool.capture.playback_speed = session_settings.get('playback_speed', 1.) width, height = session_settings.get('window_size', g_pool.capture.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = (g_pool.camera_render_size[0] + int(icon_bar_width*g_pool.gui_user_scale*hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width*g_pool.gui_user_scale*hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2.}) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width,0),size=(0,0),header_pos='hidden') g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.), size=(0., 0.), header_pos='headline') g_pool.user_timelines.color = RGBA(a=0.) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1]))) general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2]+list(np.arange(1.5, 5.1, .5)), label='Interface Size')) general_settings.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) general_settings.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) general_settings.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) general_settings.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) general_settings.append(ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100)) g_pool.export_button = ui.Thumb('export', label=chr(0xe2c5), getter=lambda: False, setter=do_export, hotkey='e', label_font='pupil_icons') g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] g_pool.plugins = Plugin_List(g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.capture.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = g_pool.capture.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.capture.play = False logger.warning("end of video") frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # pupil and gaze positions are added by their respective producer plugins events['pupil_positions'] = [] events['gaze_positions'] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture._recent_frame = frame g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = '' g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) # present frames at appropriate speed g_pool.capture.wait(frame) glfw.glfwPollEvents() session_settings['playback_speed'] = g_pool.capture.playback_speed session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.capture.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error('Process Player crashed with trace:\n{}'.format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'}) sleep(1.0)
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D, Detector_Dummy.__name__: Detector_Dummy, } # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.process = "eye{}".format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x *= hdpi_factor y *= hdpi_factor g_pool.gui.update_mouse(x, y) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, g_pool.capture.frame_size) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] plugins = (g_pool.capture_manager, g_pool.capture) # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in plugins) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id)) ) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get("flip", False) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } capture_manager_settings = session_settings.get( "capture_manager_settings", ("UVC_Manager", {}) ) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__: c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name]( g_pool, **manager_settings ) if eye_id == 0: cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"] else: cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"] # Initialize capture default_settings = ( "UVC_Source", {"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120}, ) capture_source_settings = overwrite_cap_settings or session_settings.get( "capture_settings", default_settings ) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__: c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get("roi") if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get("pupil_detector_settings", None) last_pupil_detector = pupil_detectors[ session_settings.get("last_pupil_detector", Detector_2D.__name__) ] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_ui() def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = g_pool.capture.frame_size width *= 2 height *= 2 width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2], label="Interface Size", ) ) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width *= 2 f_height *= 2 f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) def uroi_on_mouse_button(button, action, mods): if g_pool.display_mode == "roi": if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: x, y = glfw.glfwGetCursorPos(main_window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) x *= hdpi_factor y *= hdpi_factor pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size ) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size ): # if the roi interacts we dont want # the gui to interact as well return general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", ) ) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode] ) general_settings.append(g_pool.display_mode_info) detector_selector = ui.Selector( "pupil_detector", getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_Dummy, Detector_2D, Detector_3D], labels=["disabled", "C++ 2d detector", "C++ 3d detector"], label="Detection method", ) general_settings.append(detector_selector) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) toggle_general_settings(False) g_pool.pupil_detector.init_ui() g_pool.capture.init_ui() g_pool.capture_manager.init_ui() g_pool.writer = None def replace_source(source_class_name, source_settings): g_pool.capture.deinit_ui() g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) g_pool.capture.init_ui() if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "set_detection_mapping_mode": if notification["mode"] == "3d": if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True elif notification["mode"] == "2d": if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False else: if not isinstance(g_pool.pupil_detector, Detector_Dummy): set_detector(Detector_Dummy) detector_selector.read_only = True elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: record_path = notification["rec_path"] raw_mode = notification["compression"] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join( record_path, "eye{}.mp4".format(eye_id) ) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, g_pool.capture.frame_rate ) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = AV_Writer( video_path, g_pool.capture.frame_rate ) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify( { "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, } ) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif ( subject.startswith("start_eye_capture") and notification["target"] == g_pool.process ): replace_source(notification["name"], notification["args"]) elif notification["subject"].startswith("pupil_detector.set_property"): target_process = notification.get("target", g_pool.process) should_apply = target_process == g_pool.process if should_apply: try: property_name = notification["name"] property_value = notification["value"] if "2d" in notification["subject"]: g_pool.pupil_detector.set_2d_detector_property( property_name, property_value ) elif "3d" in notification["subject"]: if not isinstance(g_pool.pupil_detector, Detector_3D): raise ValueError( "3d properties are only available" " if 3d detector is active" ) g_pool.pupil_detector.set_3d_detector_property( property_name, property_value ) elif property_name == "roi": try: # Modify the ROI with the values sent over network minX, maxX, minY, maxY = property_value g_pool.u_r.set( [ max(g_pool.u_r.min_x, int(minX)), max(g_pool.u_r.min_y, int(minY)), min(g_pool.u_r.max_x, int(maxX)), min(g_pool.u_r.max_y, int(maxY)), ] ) except ValueError as err: raise ValueError( "ROI needs to be list of 4 integers:" "(minX, maxX, minY, maxY)" ) from err else: raise KeyError( "Notification subject does not " "specifiy detector type nor modify ROI." ) logger.debug( "`{}` property set to {}".format( property_name, property_value ) ) except KeyError: logger.error("Malformed notification received") logger.debug(traceback.format_exc()) except (ValueError, TypeError): logger.error("Invalid property or value") logger.debug(traceback.format_exc()) elif notification["subject"].startswith( "pupil_detector.broadcast_properties" ): target_process = notification.get("target", g_pool.process) should_respond = target_process == g_pool.process if should_respond: props = g_pool.pupil_detector.get_detector_properties() properties_broadcast = { "subject": "pupil_detector.properties.{}".format(eye_id), **props, # add properties to broadcast } ipc_socket.notify(properties_broadcast) g_pool.capture.on_notify(notification) g_pool.capture_manager.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get("frame") g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != ( f_height, f_width, ): g_pool.pupil_detector.on_resolution_change( (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size, ) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'.format( type(frame), frame_publish_format ) ) else: frame_publish_format_recent_warning = False pupil_socket.send( { "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], } ) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == "algorithm" ) if result is not None: result["id"] = eye_id result["topic"] = "pupil.{}".format(eye_id) pupil_socket.send(result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == "algorithm": g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ("camera_image", "roi"): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass glViewport(0, 0, *camera_render_size) make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame and result: if result["method"] == "3d c++": eye_ball = result["projected_sphere"] try: pts = cv2.ellipse2Poly( ( int(eye_ball["center"][0]), int(eye_ball["center"][1]), ), ( int(eye_ball["axes"][0] / 2), int(eye_ball["axes"][1] / 2), ), int(eye_ball["angle"]), 0, 360, 8, ) except ValueError as e: pass else: draw_polyline( pts, 2, RGBA(0.0, 0.9, 0.1, result["model_confidence"]), ) if result["confidence"] > 0: if "ellipse" in result: pts = cv2.ellipse2Poly( ( int(result["ellipse"]["center"][0]), int(result["ellipse"]["center"][1]), ), ( int(result["ellipse"]["axes"][0] / 2), int(result["ellipse"]["axes"][1] / 2), ), int(result["ellipse"]["angle"]), 0, 360, 15, ) confidence = result["confidence"] * 0.7 draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence)) draw_points( [result["ellipse"]["center"]], size=20, color=RGBA(1.0, 0.0, 0.0, confidence), sharpness=1.0, ) glViewport(0, 0, *camera_render_size) make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == "roi": g_pool.u_r.draw_points(g_pool.gui.scale) glViewport(0, 0, *window_size) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI unused_elements = g_pool.gui.update() for butt in unused_elements.buttons: uroi_on_mouse_button(*butt) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) g_pool.pupil_detector.visualize() # detector decides if we visualize or not # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["roi"] = g_pool.u_r.get() session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["capture_settings"] = ( g_pool.capture.class_name, g_pool.capture.get_init_dict(), ) session_settings["capture_manager_settings"] = ( g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict(), ) session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_settings[ "last_pupil_detector" ] = g_pool.pupil_detector.__class__.__name__ session_settings[ "pupil_detector_settings" ] = g_pool.pupil_detector.get_settings() session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() g_pool.capture.deinit_ui() g_pool.capture_manager.deinit_ui() g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {"subject": "set_detection_mapping_mode", "mode": new_mode} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( "1.24" ), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_routines import ( calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin, ) from fixation_detector import Fixation_Detector from eye_movement import Eye_Movement_Detector_Real_Time from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins") ) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Eye_Movement_Detector_Real_Time, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ( [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes ) plugins = ( system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins ) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ ("UVC_Source", default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), ("Screen_Marker_Calibration", {}), ("Recorder", {}), ("Pupil_Remote", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in g_pool.plugins) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world") ) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8 ) g_pool.detection_mapping_mode = session_settings.get( "detection_mapping_mode", "3d" ) g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode) def handle_notifications(noti): subject = noti["subject"] if subject == "set_detection_mapping_mode": if noti["mode"] == "2d": if ( "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name ): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(g_pool.plugin_by_name["Dummy_Gaze_Mapper"]) g_pool.detection_mapping_mode = noti["mode"] elif subject == "start_plugin": g_pool.plugins.add( g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {}) ) elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_detection_mapping_mode", "mode": g_pool.detection_mapping_mode, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) elif subject == "world_process.adapt_window_size": set_window_size() width, height = session_settings.get( "window_size", (1280 + icon_bar_width, 720) ) # window and gl setup glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0}) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.6, 0.8, 1.0, 1.2, 1.4], label="Interface size", ) ) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) on_resize(main_window, f_width, f_height) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector("audio_mode", audio, selection=audio.audio_modes) ) general_settings.append( ui.Selector( "detection_mapping_mode", g_pool, label="detection & mapping mode", setter=set_detection_mapping_mode, selection=["disabled", "2d", "3d"], ) ) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, ) ) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, ) ) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version)) ) general_settings.append( ui.Button("Restart with default settings", reset_restart) ) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins) ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins glfw.glfwPollEvents() if window_should_update() and gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString( main_window, user_input.clipboard.encode() ) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) # call `on_click` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_click(pos, button, action) for p in g_pool.plugins) for key, scancode, action, mods in user_input.keys: # call `on_key` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_key(key, scancode, action, mods) for p in g_pool.plugins) for char_ in user_input.chars: # call `char_` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_char(char_) for p in g_pool.plugins) glfw.glfwSwapBuffers(main_window) glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["detection_mapping_mode"] = g_pool.detection_mapping_mode session_settings["audio_mode"] = audio.audio_mode session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def session(rec_dir): # Callback functions def on_resize(window, w, h): g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * y_scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x] if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '%s'" % new_rec_dir) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window, True) else: logger.error("'%s' is not a valid pupil recording" % new_rec_dir) tick = delta_t() def get_dt(): return next(tick) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi') ][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") #parse info.csv file meta_info_path = os.path.join(rec_dir, "info.csv") with open(meta_info_path) as info: meta_info = dict( ((line.strip().split('\t')) for line in info.readlines())) rec_version = read_rec_version(meta_info) if rec_version >= VersionFormat('0.5'): pass elif rec_version >= VersionFormat('0.4'): update_recording_0v4_to_current(rec_dir) elif rec_version >= VersionFormat('0.3'): update_recording_0v3_to_current(rec_dir) timestamps_path = os.path.join(rec_dir, "timestamps.npy") else: logger.Error("This recording is to old. Sorry.") return timestamps = np.load(timestamps_path) # Initialize capture cap = File_Capture(video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if session_settings.get("version", VersionFormat('0.0')) < get_version(version_file): logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', (0, 0)) main_window = glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = get_version(version_file) g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.rec_version = rec_version g_pool.meta_info = meta_info g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps ] #populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: pass g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 2) except FileSeekError: pass g_pool.new_seek = True def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.main_menu = ui.Growing_Menu("Settings", pos=(-350, 20), size=(300, 400)) g_pool.main_menu.append( ui.Button("Close Pupil Player", lambda: glfwSetWindowShouldClose(main_window, True))) g_pool.main_menu.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=0.75, max=2.5, label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: %s' % g_pool.version)) g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' % rec_version)) g_pool.main_menu.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.main_menu.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize( main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label='Play', hotkey=GLFW_KEY_SPACE) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT) g_pool.backward_button = ui.Thumb('backward', getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT) g_pool.quickbar.extend( [g_pool.play_button, g_pool.forward_button, g_pool.backward_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) #we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Export_Launcher', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins + previous_plugins) for p in g_pool.plugins: if p.class_name == 'Trim_Marks': g_pool.trim_marks = p break # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) glfwSetDropCallback(main_window, on_drop) #trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp() - .03 cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" while not glfwWindowShouldClose(main_window): #grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame_nowait() except EndofVideoFileError: #end of video logic: pause at last frame. g_pool.play = False update_graph = True else: update_graph = False frame = new_frame.copy() events = {} #report time between now and the last loop interation events['dt'] = get_dt() #new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy( g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy( g_pool.pupil_positions_by_frame[frame.index]) if update_graph: #update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts != t: dt, ts = t - ts, t fps_graph.add(1. / dt) g_pool.play_button.status_text = str(frame.index) #always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() #present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.close() g_pool.gui.terminate() glfwDestroyWindow(main_window)