def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, glint_queue, glint_vector_queue): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # display import glfw from pyglui import ui, graph, cygl, __version__ as pyglui_version assert pyglui_version >= '1.3' from pyglui.cygl.utils import Named_Texture import gl_utils # monitoring import psutil # helpers/utils from version_utils import VersionFormat from file_methods import Persistent_Dict,load_object from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic from time import time import json logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin from fixation_detector import Fixation_Detector_3D from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes,Base_Manager from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.glints = glint_queue g_pool.glint_pupil_vectors = glint_vector_queue g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) calibration_plugins += [p for p in runtime_plugins if issubclass(p,Calibration_Plugin)] runtime_plugins = [p for p in runtime_plugins if not issubclass(p,Calibration_Plugin)] manager_classes += [p for p in runtime_plugins if issubclass(p,Base_Manager)] runtime_plugins = [p for p in runtime_plugins if not issubclass(p,Base_Manager)] user_launchable_plugins = [Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector_3D, Blink_Detection, Remote_Recorder] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay] plugin_by_index = (system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes + source_classes) name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_capture_settings = { 'preferred_names': ["Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e"], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {})] # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.calGlint = session_settings.get('calibrate_glint', False) g_pool.active_gaze_mapping_plugin = None g_pool.capture_manager = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def launch_eye_process(eye_id, delay=0): n = {'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay} ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = {'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode} ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) # window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (1280, 720)) main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window g_pool.world_settings = "default" def convert_keys_to_string(dictionary): """Recursively converts dictionary keys to strings.""" if not isinstance(dictionary, dict): return dictionary return dict((str(k.decode('utf8')), convert_keys_to_string(v)) for k, v in dictionary.items()) def set_world_settings(new_settings): g_pool.world_settings = new_settings if not new_settings == "default": try: path = os.path.join(g_pool.user_dir,'world_settings_' + new_settings + '.json') with open(path, 'r') as fp: json_str = fp.read() world_settings_new = json.loads(json_str) except: logger.error("Settings don't exist") return #world_settings_new = convert_keys_to_string(world_settings_new) controls = g_pool.capture.uvc_capture.controls controls_dict = dict([(c.display_name,c) for c in controls]) try: g_pool.capture.frame_rate = world_settings_new['frame_rate'] g_pool.capture.frame_size = world_settings_new['frame_size'] except: logger.info("no frame rate and frame size in camera settings") for key in controls_dict: try: controls_dict[key].value = world_settings_new[key] except: logger.info("no key with the name '%s' in camera settings" %key) def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Selector('world_settings',g_pool,setter=set_world_settings,selection=['default','outdoors'], labels=['Default', 'Outdoors'], label="World settings") ) general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,g_pool.capture.frame_size[0],g_pool.capture.frame_size[1])) ) general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes)) general_settings.append(ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d','3d'] )) general_settings.append(ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0,alive), getter=lambda: eyes_are_alive[0].value )) general_settings.append(ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1,alive), getter=lambda: eyes_are_alive[1].value )) selector_label = "Select to load" labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append(ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append(ui.Info_Text('Capture Version: {}'.format(g_pool.version))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.calibration_menu = ui.Growing_Menu('Calibration') #g_pool.calibration_menu.append(ui.Switch('calGlint', g_pool,on_val=True,off_val=False,label='Calibrate with glint')) g_pool.calibration_menu.collapsed = True g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append(g_pool.quickbar) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert(0,ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method' )) #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager', setter = open_plugin, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph g_pool.graphs = [cpu_graph, fps_graph, pupil0_graph, pupil1_graph] # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): tUnix = time() # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) #a dictionary that allows plugins to post and read events events = {} events['timestamp_unix'] = tUnix # report time between now and the last loop interation events['dt'] = get_dt() recent_glint_positions = [] while not g_pool.glints.empty(): g = g_pool.glints.get() recent_glint_positions.append(g) events['glint_positions'] = recent_glint_positions recent_glint_pupil_vectors = [] while not g_pool.glint_pupil_vectors.empty(): v = g_pool.glint_pupil_vectors.get() recent_glint_pupil_vectors.append(v) events['glint_pupil_vectors'] = recent_glint_pupil_vectors # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # update performace graphs if 'frame' in events: t = events["frame"].timestamp dt, ts = t-ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass for p in events["pupil_positions"]: pupil_graphs[p['id']].add(p['confidence']) cpu_graph.update() # send new events to ipc: del events['glint_pupil_vectors'] del events['glint_positions'] del events['timestamp_unix'] del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert(isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible(main_window): g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() for g in g_pool.graphs: g.draw() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['calibrate_glint'] = g_pool.calGlint session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.deinit_gui() # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) # shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n) zmq_ctx.destroy()
def world(timebase,eyes_are_alive,ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version,cap_src): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time,sleep import numpy as np import logging import zmq import zmq_tools #zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('pupil',)) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('notify',)) #log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #display import glfw from pyglui import ui,graph,cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen,make_coord_system_pixel_based,make_coord_system_norm_based,glFlush #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '0.8' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio from uvc import get_time_monotonic #trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin,Plugin_List,import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from pupil_sync import Pupil_Sync from pupil_remote import Pupil_Remote from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History logger.info('Application Version: %s'%version) logger.info('System Info: %s'%get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0,0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8,31) else: scroll_factor = 1.0 window_position_default = (0,0) #g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic()-g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic #manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins')) user_launchable_plugins = [Show_Calibration,Pupil_Remote,Pupil_Sync,Surface_Tracker,Annotation_Capture,Log_History]+runtime_plugins system_plugins = [Log_Display,Display_Recent_Gaze,Recorder] plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index,plugin_by_index)) default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{}),('Pupil_Remote',{})] # Callback functions def on_resize(window,w, h): if not g_pool.iconified: g_pool.gui.update_window(w,h) g_pool.gui.collect_menus() graph.adjust_size(w,h) adjust_gl_view(w,h) for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_iconify(window,iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) x,y = x*hdpi_factor,y*hdpi_factor g_pool.gui.update_mouse(x,y) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world')) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':(1280,720),'frame_rate':30} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings g_pool.iconified = False g_pool.capture = cap g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','2d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id,delay=0): n = {'subject':'eye_process.should_start.%s'%eye_id,'eye_id':eye_id,'delay':delay} ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject':'eye_process.should_stop','eye_id':eye_id} ipc_pub.notify(n) def start_stop_eye(eye_id,make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject':'set_detection_mapping_mode','mode':new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']],args=n.get('args',{}) ) elif subject == 'eye_process.started': n = {'subject':'set_detection_mapping_mode','mode':g_pool.detection_mapping_mode} ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject':'meta.doc', 'actor':g_pool.app, 'doc':world.__doc__}) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject':'meta.doc', 'actor': p.class_name, 'doc':p.on_notify.__doc__}) #window and gl setup glfw.glfwInit() width,height = session_settings.get('window_size',cap.frame_size) main_window = glfw.glfwCreateWindow(width,height, "World") window_pos = session_settings.get('window_position',window_position_default) glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes)) general_settings.append(ui.Selector('detection_mapping_mode',g_pool,label='detection & mapping mode',setter=set_detection_mapping_mode,selection=['2d','3d'])) general_settings.append(ui.Switch('eye0_process',label='Detect eye 0',setter=lambda alive: start_stop_eye(0,alive),getter=lambda: eyes_are_alive[0].value )) general_settings.append(ui.Switch('eye1_process',label='Detect eye 1',setter=lambda alive: start_stop_eye(1,alive),getter=lambda: eyes_are_alive[1].value )) general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins, labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins], setter= open_plugin, getter=lambda: "Select to load")) general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window,on_resize) glfw.glfwSetWindowIconifyCallback(main_window,on_iconify) glfw.glfwSetKeyCallback(main_window,on_key) glfw.glfwSetCharCallback(main_window,on_char) glfw.glfwSetMouseButtonCallback(main_window,on_button) glfw.glfwSetCursorPosCallback(main_window,on_pos) glfw.glfwSetScrollCallback(main_window,on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260,130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380,130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph,pupil1_graph if session_settings.get('eye1_process_alive',False): launch_eye_process(1,delay=0.6) if session_settings.get('eye0_process_alive',True): launch_eye_process(0,delay=0.3) ipc_pub.notify({'subject':'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings ipc_pub.notify({'subject':'recording.should_stop'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() recent_pupil_data = [] recent_gaze_data = [] new_notifications = [] while pupil_sub.new_data: t,p = pupil_sub.recv() pupil_graphs[p['id']].add(p['confidence']) recent_pupil_data.append(p) new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze',g) recent_gaze_data += new_gaze_data while notify_sub.new_data: t,n = notify_sub.recv() new_notifications.append(n) events['pupil_positions'] = recent_pupil_data events['gaze_positions'] = recent_gaze_data # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,events) #check if a plugin need to be destroyed g_pool.plugins.clean() #send new events to ipc: del events['pupil_positions'] #already on the wire del events['gaze_positions'] #send earlier in this loop del events['dt'] #no need to send this for topic,data in events.iteritems(): for d in data: ipc_pub.send(topic, d) # render camera image glfw.glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) glFlush() make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height,frame.width,3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil0_graph.draw() pupil1_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject':'world_process.stopped'}) #shut down launcher n = {'subject':'launcher_process.should_stop'} ipc_pub.notify(n) zmq_ctx.destroy()
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # display import glfw from pyglui import ui, graph, cygl, __version__ as pyglui_version assert pyglui_version >= '1.3' from pyglui.cygl.utils import Named_Texture import gl_utils # monitoring import psutil # helpers/utils from version_utils import VersionFormat from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin from fixation_detector import Fixation_Detector_3D from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes, Base_Manager from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) calibration_plugins += [ p for p in runtime_plugins if issubclass(p, Calibration_Plugin) ] runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, Calibration_Plugin) ] manager_classes += [ p for p in runtime_plugins if issubclass(p, Base_Manager) ] runtime_plugins = [ p for p in runtime_plugins if not issubclass(p, Base_Manager) ] user_launchable_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector_3D, Blink_Detection, Remote_Recorder ] + runtime_plugins system_plugins = [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay ] plugin_by_index = (system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes + source_classes) name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_capture_settings = { 'preferred_names': [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e" ], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {})] # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture_manager = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) # window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (1280, 720)) main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, g_pool.capture.frame_size[0], g_pool.capture. frame_size[1]))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) selector_label = "Select to load" labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append( ui.Info_Text('Capture Version: {}'.format(g_pool.version))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.calibration_menu.collapsed = True g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append(g_pool.quickbar) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert( 0, ui.Selector('capture_manager', setter=open_plugin, getter=lambda: g_pool.capture_manager.__class__, selection=manager_classes, labels=[b.gui_name for b in manager_classes], label='Manager')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph g_pool.graphs = [cpu_graph, fps_graph, pupil0_graph, pupil1_graph] # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) #a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events['dt'] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # update performace graphs if 'frame' in events: t = events["frame"].timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass for p in events["pupil_positions"]: pupil_graphs[p['id']].add(p['confidence']) cpu_graph.update() # send new events to ipc: del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible(main_window): g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() for g in g_pool.graphs: g.draw() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.deinit_gui() # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) # shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n) zmq_ctx.destroy()
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, cap_src): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time, sleep import numpy as np import logging import zmq import zmq_tools #zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('pupil', )) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) #log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush, is_window_visible #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '1.0' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio from uvc import get_time_monotonic #trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from fixation_detector import Fixation_Detector_3D from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher logger.info('Application Version: %s' % version) logger.info('System Info: %s' % get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) #g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Pupil_Groups, Frame_Publisher, Show_Calibration, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector_3D ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Fixation_Detector_3D', {})] # Callback functions def on_resize(window, w, h): if is_window_visible(window): g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (1280, 720), 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings g_pool.iconified = False g_pool.capture = cap g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '2d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.%s' % eye_id, 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) #window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', cap.frame_size) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) selector_label = "Select to load" labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings ipc_pub.notify({'subject': 'recording.should_stop'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() recent_pupil_data = [] recent_gaze_data = [] new_notifications = [] while pupil_sub.new_data: t, p = pupil_sub.recv() pupil_graphs[p['id']].add(p['confidence']) recent_pupil_data.append(p) new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze', g) recent_gaze_data += new_gaze_data while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) events['pupil_positions'] = recent_pupil_data events['gaze_positions'] = recent_gaze_data # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() #send new events to ipc: del events['pupil_positions'] #already on the wire del events['gaze_positions'] #send earlier in this loop del events['dt'] #no need to send this for topic, data in events.iteritems(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) # render camera image glfw.glfwMakeContextCurrent(main_window) if is_window_visible(main_window): g_pool.image_tex.update_from_frame(frame) glFlush() make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins if is_window_visible(main_window): for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil0_graph.draw() pupil1_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) #shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n) zmq_ctx.destroy()
def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height)
def world(pupil_queue,timebase,launcher_pipe,eye_pipes,eyes_are_alive,user_dir,version,cap_src): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) # create formatter formatter = logging.Formatter('%(processName)s - [%(levelname)s] %(name)s : %(message)s') # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir,'capture.log'),mode='w') fh.setLevel(logger.level) fh.setFormatter(formatter) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level+10) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #setup thread to recv log recrods from other processes. def log_loop(logging): import zmq ctx = zmq.Context() sub = ctx.socket(zmq.SUB) sub.bind('tcp://127.0.0.1:502020') sub.setsockopt(zmq.SUBSCRIBE, "") while True: record = sub.recv_pyobj() logger = logging.getLogger(record.name) logger.handle(record) import threading log_thread = threading.Thread(target=log_loop, args=(logging,)) log_thread.setDaemon(True) log_thread.start() # create logger for the context of this function logger = logging.getLogger(__name__) # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time,sleep import numpy as np #display import glfw from pyglui import ui,graph,cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen,make_coord_system_pixel_based,make_coord_system_norm_based #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '0.8' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio # Plug-ins from plugin import Plugin_List,import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from pupil_server import Pupil_Server from pupil_sync import Pupil_Sync from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from pupil_remote import Pupil_Remote from log_history import Log_History from game_controller import GameController logger.info('Application Version: %s'%version) logger.info('System Info: %s'%get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0,0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8,31) else: scroll_factor = 1.0 window_position_default = (0,0) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # g_pool.launcher_pipe = launcher_pipe g_pool.eye_pipes = eye_pipes g_pool.eyes_are_alive = eyes_are_alive #manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins')) user_launchable_plugins = [GameController, Show_Calibration,Pupil_Remote,Pupil_Server,Pupil_Sync,Surface_Tracker,Annotation_Capture,Log_History]+runtime_plugins system_plugins = [Log_Display,Display_Recent_Gaze,Recorder] plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index,plugin_by_index)) default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})] # Callback functions def on_resize(window,w, h): if not g_pool.iconified: g_pool.gui.update_window(w,h) g_pool.gui.collect_menus() graph.adjust_size(w,h) adjust_gl_view(w,h) for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_iconify(window,iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) x,y = x*hdpi_factor,y*hdpi_factor g_pool.gui.update_mouse(x,y) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world')) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':(1280,720),'frame_rate':30} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() launcher_pipe.send("Exit") return g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6) g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','2d') g_pool.active_calibration_plugin = None audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id,blocking=False): if eyes_are_alive[eye_id].value: logger.error("Eye%s process already running."%eye_id) return launcher_pipe.send(eye_id) eye_pipes[eye_id].send( ('Set_Detection_Mapping_Mode',g_pool.detection_mapping_mode) ) if blocking: #wait for ready message from eye to sequentialize startup eye_pipes[eye_id].send('Ping') eye_pipes[eye_id].recv() logger.warning('Eye %s process started.'%eye_id) def stop_eye_process(eye_id,blocking=False): if eyes_are_alive[eye_id].value: eye_pipes[eye_id].send('Exit') if blocking: while eyes_are_alive[eye_id].value: sleep(.1) def start_stop_eye(eye_id,make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): if new_mode == '2d': for p in g_pool.plugins: if "Vector_Gaze_Mapper" in p.class_name: logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") p.alive = False g_pool.plugins.clean() for alive, pipe in zip(g_pool.eyes_are_alive,g_pool.eye_pipes): if alive.value: pipe.send( ('Set_Detection_Mapping_Mode',new_mode) ) g_pool.detection_mapping_mode = new_mode #window and gl setup glfw.glfwInit() width,height = session_settings.get('window_size',(frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width,height, "World") window_pos = session_settings.get('window_position',window_position_default) glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes)) general_settings.append(ui.Selector('detection_mapping_mode',g_pool,label='detection & mapping mode',setter=set_detection_mapping_mode,selection=['2d','3d'])) general_settings.append(ui.Switch('eye0_process',label='Detect eye 0',setter=lambda alive: start_stop_eye(0,alive),getter=lambda: eyes_are_alive[0].value )) general_settings.append(ui.Switch('eye1_process',label='Detect eye 1',setter=lambda alive: start_stop_eye(1,alive),getter=lambda: eyes_are_alive[1].value )) general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins, labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins], setter= open_plugin, getter=lambda: "Select to load")) general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence')) general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window,on_resize) glfw.glfwSetWindowIconifyCallback(main_window,on_iconify) glfw.glfwSetKeyCallback(main_window,on_key) glfw.glfwSetCharCallback(main_window,on_char) glfw.glfwSetMouseButtonCallback(main_window,on_button) glfw.glfwSetCursorPosCallback(main_window,on_pos) glfw.glfwSetScrollCallback(main_window,on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260,130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" if session_settings.get('eye1_process_alive',False): launch_eye_process(1,blocking=True) if session_settings.get('eye0_process_alive',True): launch_eye_process(0,blocking=False) # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings g_pool.notifications.append({'subject':'should_stop_recording'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height,frame.width,3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0,blocking = True) stop_eye_process(1,blocking = True) #shut down laucher launcher_pipe.send("Exit") logger.info("Process Shutting down.")
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url,ipc_push_url, user_dir, version, eye_id,overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=("notify",)) with Is_Alive_Manager(is_alive_flag,ipc_socket,eye_id): #logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #general imports import numpy as np import cv2 #display import glfw from pyglui import ui,graph,cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere import OpenGL.GL as gl from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based,is_window_visible from ui_roi import UIRoi #monitoring import psutil import math # helpers/utils from uvc import get_time_monotonic, StreamError from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from av_writer import JPEG_Writer,AV_Writer from video_capture import InitialisationError,StreamError, Fake_Source,EndofVideoFileError, source_classes, manager_classes source_by_name = {src.class_name():src for src in source_classes} # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D} #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600,300*eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600,31+300*eye_id) else: scroll_factor = 1.0 window_position_default = (600,300*eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic()-g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window,w, h): if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w,h) graph.adjust_size(w,h) adjust_gl_view(w,h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_iconify(window,iconified): g_pool.iconified = iconified def on_button(window,button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button,action,mods) def on_pos(window,x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x,y),glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height) ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id)) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager',{})) if eye_id == 0: cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"] else: cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"] # Initialize capture default_settings = { 'source_class_name': 'UVC_Source', 'preferred_names' : cap_src, 'frame_size': (640,480), 'frame_rate': 90 } settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings) try: cap = source_by_name[settings['source_class_name']](g_pool, **settings) except (KeyError,InitialisationError) as e: if isinstance(e,KeyError): logger.warning('Incompatible capture setting encountered. Falling back to fake source.') cap = Fake_Source(g_pool, **settings) g_pool.iconified = False g_pool.capture = cap g_pool.capture_manager = None g_pool.flip = session_settings.get('flip',False) g_pool.display_mode = session_settings.get('display_mode','camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) writer = None pupil_detector_settings = session_settings.get('pupil_detector_settings',None) last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "eye %s"%eye_id width,height = session_settings.get('window_size',g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width,height, title, None, None) window_pos = session_settings.get('window_position',window_position_default) glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() glfw.glfwSwapInterval(0) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture.init_gui() g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.pupil_detector.init_gui(g_pool.sidebar) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__:c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings) g_pool.capture_manager.init_gui() def open_manager(manager_class): g_pool.capture_manager.cleanup() g_pool.capture_manager = manager_class(g_pool) g_pool.capture_manager.init_gui() #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager',g_pool, setter = open_manager, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window,on_resize) glfw.glfwSetWindowIconifyCallback(main_window,on_iconify) glfw.glfwSetKeyCallback(main_window,on_key) glfw.glfwSetCharCallback(main_window,on_char) glfw.glfwSetMouseButtonCallback(main_window,on_button) glfw.glfwSetCursorPosCallback(main_window,on_pos) glfw.glfwSetScrollCallback(main_window,on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" should_publish_frames = False frame_publish_format = 'jpeg' #create a timer to control window update frequency window_update_timer = timer(1/60.) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t,notification = notify_sub.recv() subject = notification['subject'] if subject == 'eye_process.should_stop': if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector,Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector,Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye']: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info("Will save eye video to: %s"%record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = JPEG_Writer(video_path,g_pool.capture.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = AV_Writer(video_path,g_pool.capture.frame_rate) timestamps = [] elif subject == 'recording.stopped': if writer: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path,np.asarray(timestamps)) del timestamps elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject':'meta.doc', 'actor':'eye%i'%eye_id, 'doc':eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format','jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' else: g_pool.capture_manager.on_notify(notification) # Get an image from the grabber try: frame = g_pool.capture.get_frame() except StreamError as e: logger.error("Error getting frame. Stopping eye process.") logger.debug("Caught error: %s"%e) break except EndofVideoFileError: logger.warning("Video File is done. Stopping") g_pool.capture.seek_to_frame(0) frame = g_pool.capture.get_frame() g_pool.u_r = UIRoi((frame.height,frame.width)) g_pool.capture_manager.update(frame, {}) if should_publish_frames and frame.jpeg_buffer: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray pupil_socket.send('frame.eye.%s'%eye_id,{ 'width': frame.width, 'height': frame.width, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s'%eye_id,result) # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image','roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() window_size = glfw.glfwGetWindowSize(main_window) make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip) g_pool.capture.gl_display() if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) ) if result['confidence'] >0: if result.has_key('ellipse'): pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)), int(result['ellipse']['angle']),0,360,15) confidence = result['confidence'] * 0.7 #scale it a little draw_polyline(pts,1,RGBA(1.,0,0,confidence)) draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() #detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path,np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict() session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_gui() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() logger.info("Process shutting down.")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source, EndofVideoFileError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from batch_exporter import Batch_Exporter, Batch_Export from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs assert VersionFormat(pyglui_version) >= VersionFormat('1.13'), 'pyglui out of date, please upgrade to newest version' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export] user_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Offline_Fixation_Detector, Batch_Exporter, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None # sets itself to g_pool.capture File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', g_pool.capture.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = (g_pool.camera_render_size[0] + int(icon_bar_width*g_pool.gui_user_scale*hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width*g_pool.gui_user_scale*hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin # def next_frame(_): # try: # g_pool.capture.seek_to_frame(g_pool.capture.get_frame_index() + 1) # except(FileSeekError): # logger.warning("Could not seek to next frame.") # else: # g_pool.new_seek = True # def prev_frame(_): # try: # g_pool.capture.seek_to_frame(g_pool.capture.get_frame_index() - 1) # except(FileSeekError): # logger.warning("Could not seek to previous frame.") # else: # g_pool.new_seek = True # def toggle_play(new_state): # if g_pool.capture.get_frame_index() >= g_pool.capture.get_frame_count()-5: # g_pool.capture.seek_to_frame(1) # avoid pause set by hitting trimmark pause. # logger.warning("End of video - restart at beginning.") # g_pool.capture.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2.}) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width,0),size=(0,0),header_pos='hidden') g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.), size=(0., 0.), header_pos='headline') g_pool.user_timelines.color = RGBA(a=0.) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1]))) general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2]+list(np.arange(1.5, 5.1, .5)), label='Interface Size')) general_settings.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) general_settings.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) general_settings.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) general_settings.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) general_settings.append(ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100)) g_pool.export_button = ui.Thumb('export', label=chr(0xe2c4), getter=lambda: False, setter=do_export, hotkey='e', label_font='pupil_icons') g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] g_pool.plugins = Plugin_List(g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.capture.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = g_pool.capture.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.capture.play = False logger.warning("end of video") frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # pupil and gaze positions are added by their respective producer plugins events['pupil_positions'] = [] events['gaze_positions'] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture._recent_frame = frame g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) unused_elements = g_pool.gui.update() for b in unused_elements.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) # present frames at appropriate speed g_pool.capture.wait(frame) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.capture.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error('Process Player crashed with trace:\n{}'.format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'}) sleep(1.0)
def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = (g_pool.camera_render_size[0] + int(icon_bar_width*g_pool.gui_user_scale*hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width*g_pool.gui_user_scale*hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size)
def world(pupil_queue, timebase, launcher_pipe, eye_pipes, eyes_are_alive, user_dir, version, cap_src): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) # create formatter formatter = logging.Formatter( '%(processName)s - [%(levelname)s] %(name)s : %(message)s') # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir, 'capture.log'), mode='w') fh.setLevel(logger.level) fh.setFormatter(formatter) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #setup thread to recv log recrods from other processes. def log_loop(logging): import zmq ctx = zmq.Context() sub = ctx.socket(zmq.SUB) sub.bind('tcp://127.0.0.1:502020') sub.setsockopt(zmq.SUBSCRIBE, "") while True: record = sub.recv_pyobj() logger = logging.getLogger(record.name) logger.handle(record) import threading log_thread = threading.Thread(target=log_loop, args=(logging, )) log_thread.setDaemon(True) log_thread.start() # create logger for the context of this function logger = logging.getLogger(__name__) # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time, sleep import numpy as np #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '0.8' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio # Plug-ins from plugin import Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from pupil_server import Pupil_Server from pupil_sync import Pupil_Sync from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from pupil_remote import Pupil_Remote from log_history import Log_History logger.info('Application Version: %s' % version) logger.info('System Info: %s' % get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # g_pool.launcher_pipe = launcher_pipe g_pool.eye_pipes = eye_pipes g_pool.eyes_are_alive = eyes_are_alive #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Show_Calibration, Pupil_Remote, Pupil_Server, Pupil_Sync, Surface_Tracker, Annotation_Capture, Log_History ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {})] # Callback functions def on_resize(window, w, h): if not g_pool.iconified: g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (1280, 720), 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() launcher_pipe.send("Exit") return g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get( 'pupil_confidence_threshold', .6) g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '2d') g_pool.active_calibration_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: logger.error("Eye%s process already running." % eye_id) return launcher_pipe.send(eye_id) eye_pipes[eye_id].send( ('Set_Detection_Mapping_Mode', g_pool.detection_mapping_mode)) if blocking: #wait for ready message from eye to sequentialize startup eye_pipes[eye_id].send('Ping') eye_pipes[eye_id].recv() logger.warning('Eye %s process started.' % eye_id) def stop_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: eye_pipes[eye_id].send('Exit') if blocking: while eyes_are_alive[eye_id].value: sleep(.1) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): if new_mode == '2d': for p in g_pool.plugins: if "Vector_Gaze_Mapper" in p.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) p.alive = False g_pool.plugins.clean() for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes): if alive.value: pipe.send(('Set_Detection_Mapping_Mode', new_mode)) g_pool.detection_mapping_mode = new_mode #window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) general_settings.append( ui.Slider('pupil_confidence_threshold', g_pool, step=.01, min=0., max=1., label='Minimum pupil confidence')) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" if session_settings.get('eye1_process_alive', False): launch_eye_process(1, blocking=True) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, blocking=False) # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings g_pool.notifications.append({'subject': 'should_stop_recording'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings[ 'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0, blocking=True) stop_eye_process(1, blocking=True) #shut down laucher launcher_pipe.send("Exit") logger.info("Process Shutting down.")
def main(): # verify input if len(sys.argv) == 1: # check if the standard input is a tty (not a pipe) if sys.stdin.isatty(): print("Incorrect syntax, use:") print(' > ' + sys.argv[0] + " image.png") # show a default image if exists sys.argv.append('/Users/facciolo/uiskentuie_standing_stone.png') try: from os import stat stat(sys.argv[1]) except OSError: sys.exit(1) # otherwise use stdin as input (because it should be a pipe) else: sys.argv.append('-') # pick the first image I1 = sys.argv[1] # globals global D,V,DD,current_image_idx tic() # read the image # Usually this is done with change_image, but this is a special case # as we must find out the image size before creating the window D.imageBitmapTiles,D.w,D.h,D.nch,D.v_min,D.v_max = load_image(I1) D.filename = I1 from os import stat, path if I1 != '-' and path.exists(I1): D.mtime = (stat(I1).st_mtime) V.data_min, V.data_max= D.v_min,D.v_max V.reset_scale_bias() toc('loadImage+data->RGBbitmap') ## image list current_image_idx = 0 DD[current_image_idx] = D tic() # Initialize the library # for some reason glfwInit changes the current dir, so I change it back! from os import getcwd,chdir savepath = getcwd() if not glfw.glfwInit(): sys.exit(1) chdir(savepath) # TODO REMOVE : needed for the text import OpenGL.GLUT as glut glut.glutInit() # Create a windowed mode window and its OpenGL context window = glfw.glfwCreateWindow(D.w, D.h, "Vflip! (reloaded)", None, None) if not window: glfw.glfwTerminate() sys.exit(1) # Make the window's context current glfw.glfwMakeContextCurrent(window) # event handlers glfw.glfwSetKeyCallback(window, keyboard_callback) glfw.glfwSetMouseButtonCallback(window, mouseButtons_callback) glfw.glfwSetScrollCallback(window, mouseWheel_callback) glfw.glfwSetCursorPosCallback(window, mouseMotion_callback) glfw.glfwSetFramebufferSizeCallback(window, resize_callback) glfw.glfwSetWindowRefreshCallback(window,display) # glfw.glfwSetCharCallback (window, unicode_char_callback) toc('glfw init') # setup texture tic() setupTexturesFromImageTiles(D.imageBitmapTiles,D.w,D.h,D.nch) glFinish() # Flush and wait toc('texture setup') print (0,D.filename, (D.w,D.h,D.nch), (D.v_min,D.v_max)) ########## ######## SETUP FRAGMENT SHADER FOR CONTRAST CHANGE ########## # http://www.cityinabottle.org/nodebox/shader/ # http://www.seethroughskin.com/blog/?p=771 # http://python-opengl-examples.blogspot.com.es/ # http://www.lighthouse3d.com/tutorials/glsl-core-tutorial/fragment-shader/ global program, SHADERS, SHADER_PROGRAMS for s in SHADERS.keys(): SHADER_PROGRAMS[s] = compileProgram( compileShader(SHADERS[s], GL_FRAGMENT_SHADER), ); glLinkProgram( SHADER_PROGRAMS[s] ) program = SHADER_PROGRAMS['rgba'] #global program # try to activate/enable shader program # handle errors wisely try: glUseProgram(program) except OpenGL.error.GLError: print(glGetProgramInfoLog(program)) raise # set the values of the shader uniform variables (global) shader_a= glGetUniformLocation(program, "shader_a") glUniform1f(shader_a,V.scale_param) shader_b= glGetUniformLocation(program, "shader_b") glUniform1f(shader_b,V.bias_param) shader_c= glGetUniformLocation(program, "shader_c") glUniform1i(shader_c,V.inv_param) # # first display # display(window) ## glFlush() # Loop until the user closes the window while not glfw.glfwWindowShouldClose(window): # Render here if V.redisp: # Try to resize the window if needed # this process the window resize requests generated by the application # the user window resize requests requests go directly to resize_callback if V.resize and not (D.w,D.h) == glfw.glfwGetFramebufferSize(window) and not V.window_has_been_resized_by_the_user: glfw.glfwSetWindowSize(window,D.w,D.h) # I know it's not been the user so I reset the variable to 0 V.window_has_been_resized_by_the_user=0 V.resize = 0 if V.TOGGLE_FIT_TO_WINDOW_SIZE: V.update_zoom_position_to_fit_window() V.redisp = display(window) # Swap front and back buffers glfw.glfwSwapBuffers(window) V.mute_wheel=0 V.mute_sweep=0 V.mute_keyboard=0 # Poll for and process events #glfw.glfwPollEvents() glfw.glfwWaitEvents() glfw.glfwTerminate()
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id): # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # general imports import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D } # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.process = 'eye{}'.format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size + 40, g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0] g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, g_pool.capture.frame_size) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id))) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below." } capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager', {})) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__: c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name]( g_pool, **manager_settings) if eye_id == 0: cap_src = [ "Pupil Cam1 ID0", "HD-6000", "Integrated Camera", "HD USB Camera", "USB 2.0 Camera" ] else: cap_src = ["Pupil Cam1 ID1", "HD-6000", "Integrated Camera"] # Initialize capture default_settings = ('UVC_Source', { 'preferred_names': cap_src, 'frame_size': (640, 480), 'frame_rate': 90 }) capture_source_settings = overwrite_cap_settings or session_settings.get( 'capture_settings', default_settings) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__: c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings) assert g_pool.capture g_pool.u_r = UIRoi( (g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get('window_size', g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray( np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, *g_pool.capture.frame_size))) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector( 'pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.capture.init_gui() g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.pupil_detector.init_gui(g_pool.sidebar) g_pool.capture_manager.init_gui() g_pool.writer = None def replace_source(source_class_name, source_settings): g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings) g_pool.capture.init_gui() if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture def replace_manager(manager_class): g_pool.capture_manager.cleanup() g_pool.capture_manager = manager_class(g_pool) g_pool.capture_manager.init_gui() #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert( 0, ui.Selector('capture_manager', g_pool, setter=replace_manager, getter=lambda: g_pool.capture_manager.__class__, selection=manager_classes, labels=[b.gui_name for b in manager_classes], label='Manager')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = 'jpeg' # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject == 'eye_process.should_stop': if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye'] and g_pool.capture.online: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info( "Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, g_pool.capture.frame_rate) elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate) else: g_pool.writer = AV_Writer( video_path, g_pool.capture.frame_rate) elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye{}'.format(eye_id), 'doc': eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format', 'jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' elif subject.startswith('start_eye_capture') and notification[ 'target'] == g_pool.process: replace_source(notification['name'], notification['args']) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get('frame') g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width): g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames and frame.jpeg_buffer: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray pupil_socket.send( 'frame.eye.%s' % eye_id, { 'width': frame.width, 'height': frame.width, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s' % eye_id, result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame: if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline( pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if 'ellipse' in result: pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result['confidence'] * 0.7 draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI g_pool.gui.update() # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize( ) # detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings[ 'capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict( ) session_settings[ 'capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict( ) session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings[ 'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings[ 'pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_gui() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() logger.info("Process shutting down.")
def _set_initial_window_state(self): glfw.glfwSetWindowPos(self._window, *self._general_settings.window_position) glfw.glfwSetWindowSize(self._window, *self._general_settings.window_size)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from file_methods import Persistent_Dict, load_object import numpy as np # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils # capture from video_capture import File_Source, EndofVideoFileError, FileSeekError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # monitoring import psutil # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_bar import Seek_Bar from trim_marks import Trim_Marks from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector # from manual_gaze_correction import Manual_Gaze_Correction from batch_exporter import Batch_Exporter from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration assert pyglui_version >= '1.5' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.get_timestamp = lambda: 0. g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index() + 1) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 1) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) g_pool.main_menu.append(ui.Info_Text('Open plugins')) selector_label = "Select to load" def append_selector(label, plugins): plugins.sort(key=lambda p: p.__name__) plugin_labels = [p.__name__.replace('_', ' ') for p in plugins] g_pool.main_menu.append(ui.Selector(label, selection=[selector_label] + plugins, labels=[selector_label] + plugin_labels, setter=open_plugin, getter=lambda: selector_label)) base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base] base_labels = ['Visualizer:', 'Analyser:', 'Data Source:'] launchable = user_launchable_plugins.copy() for base_class, label in zip(base_plugins, base_labels): member_plugins = [] for p in user_launchable_plugins: if issubclass(p, base_class): member_plugins.append(p) launchable.remove(p) append_selector(label, member_plugins) # launchable only contains plugins that could not be assigned to any of the above categories append_selector('Other', launchable) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=glfw.GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=glfw.GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=glfw.GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() unused_buttons = g_pool.gui.update() for b in unused_buttons: button,action,mods = b pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) # present frames at appropriate speed cap.wait(frame) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'})
def set_window_size(): glfw.glfwSetWindowSize(main_window, *window_size_default)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ is_alive = Is_Alive_Manager(is_alive_flag) with is_alive: import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id), mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'Eye' + str(eye_id) + ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'EYE' + str(eye_id) + ' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) logging.getLogger("libav").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. #general imports import numpy as np import cv2 #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture from OpenGL.GL import GL_LINE_LOOP from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based from ui_roi import UIRoi #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from av_writer import JPEG_Writer, AV_Writer # Pupil detectors from pupil_detectors import Canny_Detector #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600, 31 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # Callback functions def on_resize(window, w, h): if not g_pool.iconified: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and u_r.active_edit_pt: u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40, u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) u_r.move_vertex(u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (640, 480), 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go # pipe_to_world.send('eye%s process ready'%eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below." } # g_pool.draw_pupil = session_settings.get('draw_pupil',True) u_r = UIRoi(frame.img.shape) u_r.set(session_settings.get('roi', u_r.get())) writer = None pupil_detector = Canny_Detector(g_pool) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] # Initialize glfw glfw.glfwInit() title = "eye %s" % eye_id width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfw.glfwSwapInterval(0) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # let detector add its GUI pupil_detector.init_gui(g_pool.sidebar) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1 / 60.) def window_should_update(): return next(window_update_timer) # Event loop while not glfw.glfwWindowShouldClose(main_window): if pipe_to_world.poll(): command = pipe_to_world.recv() if command == 'Exit': break elif command == "Ping": pipe_to_world.send("Pong") command = None else: command = None # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if command: if 'Rec_Start:' in command: record_path, raw_mode = command logger.info("Will save eye video to: %s" % record_path) timestamps_path = os.path.join( record_path, "eye%s_timestamps.npy" % eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = JPEG_Writer(video_path, cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4" % eye_id) writer = AV_Writer(video_path, cap.frame_rate) timestamps = [] elif 'Rec_Stop:' in command: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect( frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() # switch to work in pixel space make_coord_system_pixel_based( (frame.height, frame.width, 3), g_pool.flip) if result['confidence'] > 0: if result.has_key('axes'): pts = cv2.ellipse2Poly( (int(result['center'][0]), int(result['center'][1])), (int(result['axes'][0] / 2), int(result['axes'][1] / 2)), int(result['angle']), 0, 360, 15) draw_polyline(pts, 1, RGBA(1., 0, 0, .5)) draw_points([result['center']], size=20, color=RGBA(1., 0., 0., .5), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': u_r.draw(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = g_pool.version session_settings.close() pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() logger.debug("Process done")
def world(pupil_queue, timebase, lauchner_pipe, eye_pipes, eyes_are_alive, user_dir, version, cap_src): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir, "world.log"), mode="w") fh.setLevel(logger.level) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter("World Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s") fh.setFormatter(formatter) formatter = logging.Formatter("WORLD Process [%(levelname)s] %(name)s : %(message)s") ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) logging.getLogger("libav").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmfull but unnessasary. # general imports from time import time import numpy as np # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture from gl_utils import ( basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, ) # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= "0.7" # monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat # Plug-ins from plugin import Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from pupil_server import Pupil_Server from pupil_sync import Pupil_Sync from marker_detector import Marker_Detector from log_display import Log_Display from annotations import Annotation_Capture # create logger for the context of this function # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == "Windows": scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # g_pool.lauchner_pipe = lauchner_pipe g_pool.eye_pipes = eye_pipes g_pool.eyes_are_alive = eyes_are_alive # manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, "plugins")) user_launchable_plugins = [ Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector, Annotation_Capture, ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [ ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), ("Screen_Marker_Calibration", {}), ("Recorder", {}), ] # Callback functions def on_resize(window, w, h): if not g_pool.iconified: g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, "user_settings_world")) if session_settings.get("version", VersionFormat("0.0")) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {"frame_size": (1280, 720), "frame_rate": 30} previous_settings = session_settings.get("capture_settings", None) if previous_settings and previous_settings["name"] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() lauchner_pipe.send("Exit") return g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get("pupil_confidence_threshold", 0.6) g_pool.detection_mapping_mode = session_settings.get("detection_mapping_mode", "2d") g_pool.active_calibration_plugin = None def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: logger.error("Eye%s process already running." % eye_id) return lauchner_pipe.send(eye_id) eye_pipes[eye_id].send(("Set_Detection_Mapping_Mode", g_pool.detection_mapping_mode)) if blocking: # wait for ready message from eye to sequentialize startup eye_pipes[eye_id].send("Ping") eye_pipes[eye_id].recv() logger.warning("Eye %s process started." % eye_id) def stop_eye_process(eye_id, blocking=False): if eyes_are_alive[eye_id].value: eye_pipes[eye_id].send("Exit") if blocking: raise NotImplementedError() def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): if new_mode == "2d": for p in g_pool.plugins: if "Vector_Gaze_Mapper" in p.class_name: logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") p.alive = False g_pool.plugins.clean() for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes): if alive.value: pipe.send(("Set_Detection_Mapping_Mode", new_mode)) g_pool.detection_mapping_mode = new_mode # window and gl setup glfw.glfwInit() width, height = session_settings.get("window_size", (frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window # setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get("gui_scale", 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos="left") general_settings = ui.Growing_Menu("General") general_settings.append( ui.Slider("scale", g_pool.gui, setter=set_scale, step=0.05, min=1.0, max=2.5, label="Interface size") ) general_settings.append( ui.Button("Reset window size", lambda: glfw.glfwSetWindowSize(main_window, frame.width, frame.height)) ) general_settings.append( ui.Selector( "detection_mapping_mode", g_pool, label="detection & mapping mode", setter=set_detection_mapping_mode, selection=["2d", "3d"], ) ) general_settings.append( ui.Switch( "eye0_process", label="detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value, ) ) general_settings.append( ui.Switch( "eye1_process", label="detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value, ) ) general_settings.append( ui.Selector( "Open plugin", selection=user_launchable_plugins, labels=[p.__name__.replace("_", " ") for p in user_launchable_plugins], setter=open_plugin, getter=lambda: "Select to load", ) ) general_settings.append( ui.Slider("pupil_confidence_threshold", g_pool, step=0.01, min=0.0, max=1.0, label="Minimum pupil confidence") ) general_settings.append(ui.Info_Text("Capture Version: %s" % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu("Calibration") g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) # plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get("loaded_plugins", default_plugins)) # We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( "active_calibration_plugin", getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace("_", " ") for p in calibration_plugins], setter=open_plugin, label="Method", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfw.glfwSwapInterval(0) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" if session_settings.get("eye1_process_alive", False): launch_eye_process(1, blocking=True) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, blocking=False) # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Stopping.") break except EndofVideoFileError: logger.warning("Video file is done. Stopping") break # update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass cpu_graph.update() # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p["confidence"]) events["pupil_positions"] = recent_pupil_positions # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n["_notify_time_"] < time(): del n["_notify_time_"] del g_pool.delayed_notifications[n["subject"]] g_pool.notifications.append(n) # notify each plugin if there are new notifications: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["pupil_confidence_threshold"] = g_pool.pupil_confidence_threshold session_settings["gui_scale"] = g_pool.gui.scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["capture_settings"] = g_pool.capture.settings session_settings["window_size"] = glfw.glfwGetWindowSize(main_window) session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = g_pool.version session_settings["eye0_process_alive"] = eyes_are_alive[0].value session_settings["eye1_process_alive"] = eyes_are_alive[1].value session_settings["detection_mapping_mode"] = g_pool.detection_mapping_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() # shut down eye processes: stop_eye_process(0) stop_eye_process(1) # shut down laucher lauchner_pipe.send("Exit") logger.debug("world process done")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from file_methods import Persistent_Dict, load_object import numpy as np # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils # capture from video_capture import File_Source, EndofVideoFileError, FileSeekError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # monitoring import psutil # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_bar import Seek_Bar from trim_marks import Trim_Marks from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector # from manual_gaze_correction import Manual_Gaze_Correction from batch_exporter import Batch_Exporter from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration assert pyglui_version >= '1.6' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.get_timestamp = lambda: 0. g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index() + 1) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 1) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) g_pool.main_menu.append(ui.Info_Text('Open plugins')) selector_label = "Select to load" def append_selector(label, plugins): plugins.sort(key=lambda p: p.__name__) plugin_labels = [p.__name__.replace('_', ' ') for p in plugins] g_pool.main_menu.append(ui.Selector(label, selection=[selector_label] + plugins, labels=[selector_label] + plugin_labels, setter=open_plugin, getter=lambda: selector_label)) base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base] base_labels = ['Visualizer:', 'Analyser:', 'Data Source:'] launchable = user_launchable_plugins.copy() for base_class, label in zip(base_plugins, base_labels): member_plugins = [] for p in user_launchable_plugins: if issubclass(p, base_class): member_plugins.append(p) launchable.remove(p) append_selector(label, member_plugins) # launchable only contains plugins that could not be assigned to any of the above categories append_selector('Other', launchable) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=glfw.GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=glfw.GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=glfw.GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() unused_elements = g_pool.gui.update() for b in unused_elements.buttons: button, action, mods = b pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) # present frames at appropriate speed cap.wait(frame) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'})
def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) on_resize(main_window, f_width, f_height)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ is_alive = Is_Alive_Manager(is_alive_flag) with is_alive: import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level+10) # create formatter and add it to the handlers formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. #general imports import numpy as np import cv2 #display import glfw from pyglui import ui,graph,cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere import OpenGL.GL as gl from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based from ui_roi import UIRoi #monitoring import psutil import math # helpers/utils from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from av_writer import JPEG_Writer,AV_Writer # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D} #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600,300*eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600,31+300*eye_id) else: scroll_factor = 1.0 window_position_default = (600,300*eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # Callback functions def on_resize(window,w, h): if not g_pool.iconified: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w,h) graph.adjust_size(w,h) adjust_gl_view(w,h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_iconify(window,iconified): g_pool.iconified = iconified def on_button(window,button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button,action,mods) def on_pos(window,x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x,y),glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height) ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id)) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':(640,480),'frame_rate':60} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go # pipe_to_world.send('eye%s process ready'%eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip',False) g_pool.display_mode = session_settings.get('display_mode','camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} g_pool.u_r = UIRoi(frame.img.shape) g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get())) def on_frame_size_change(new_size): g_pool.u_r = UIRoi((new_size[1],new_size[0])) cap.on_frame_size_change = on_frame_size_change writer = None pupil_detector_settings = session_settings.get('pupil_detector_settings',None) last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "eye %s"%eye_id width,height = session_settings.get('window_size',(frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width,height, title, None, None) window_pos = session_settings.get('window_position',window_position_default) glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfw.glfwSwapInterval(0) sphere = Sphere(20) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) # let detector add its GUI g_pool.pupil_detector.init_gui(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window,on_resize) glfw.glfwSetWindowIconifyCallback(main_window,on_iconify) glfw.glfwSetKeyCallback(main_window,on_key) glfw.glfwSetCharCallback(main_window,on_char) glfw.glfwSetMouseButtonCallback(main_window,on_button) glfw.glfwSetCursorPosCallback(main_window,on_pos) glfw.glfwSetScrollCallback(main_window,on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1/60.) def window_should_update(): return next(window_update_timer) # Event loop while not glfw.glfwWindowShouldClose(main_window): if pipe_to_world.poll(): cmd = pipe_to_world.recv() if cmd == 'Exit': break elif cmd == "Ping": pipe_to_world.send("Pong") command = None else: command,payload = cmd if command == 'Set_Detection_Mapping_Mode': if payload == '3d': if not isinstance(g_pool.pupil_detector,Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: set_detector(Detector_2D) detector_selector.read_only = False else: command = None # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") cap.seek_to_frame(0) frame = cap.get_frame() #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if 'Rec_Start' == command: record_path,raw_mode = payload logger.info("Will save eye video to: %s"%record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = JPEG_Writer(video_path,cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = AV_Writer(video_path,cap.frame_rate) timestamps = [] elif 'Rec_Stop' == command: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path,np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image','roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() window_size = glfw.glfwGetWindowSize(main_window) make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip) if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) ) if result['confidence'] >0: if result.has_key('ellipse'): pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)), int(result['ellipse']['angle']),0,360,15) confidence = result['confidence'] * 0.7 #scale it a little draw_polyline(pts,1,RGBA(1.,0,0,confidence)) draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': g_pool.u_r.draw(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() #detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path,np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() logger.debug("Process done")
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, glint_queue, glint_vector_queue, overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id): # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # general imports import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible from ui_roi import UIRoi # monitoring import psutil import math from time import time import json # helpers/utils from file_methods import Persistent_Dict, load_object # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D} # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (600,31+ 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.process = 'eye{}'.format(eye_id) g_pool.timebase = timebase g_pool.glints = glint_queue g_pool.glint_pupil_vectors = glint_vector_queue glint_detector = Glint_Detector(g_pool) g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0] g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,g_pool.capture.frame_size ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id))) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get( 'display_mode', 'camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager',{})) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__:c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings) pupil_detector_settings = session_settings.get('pupil_detector_settings',None) last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings) g_pool.pupil_settings = "default" if eye_id == 0: cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"] else: cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"] # Initialize capture default_settings = ('UVC_Source',{ 'preferred_names' : cap_src, 'frame_size': (640,480), 'frame_rate': 90 }) capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__:c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector( g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) def convert_keys_to_string(dictionary): """Recursively converts dictionary keys to strings.""" if not isinstance(dictionary, dict): return dictionary return dict((str(k.decode('utf8')), convert_keys_to_string(v)) for k, v in dictionary.items()) def set_pupil_settings(new_settings): g_pool.pupil_settings = new_settings if not new_settings == "default": try: path = os.path.join(g_pool.user_dir,'pupil_settings_' + new_settings + '.json') with open(path, 'r') as fp: json_str = fp.read() pupil_settings_new = json.loads(json_str) except: logger.error("Settings don't exist") #pupil_settings_new = convert_keys_to_string(pupil_settings_new) pupil_settings_new['2D_Settings'] = dict(pupil_settings_new) pupil_settings = g_pool.pupil_detector.get_settings() controls = g_pool.capture.uvc_capture.controls controls_dict = dict([(c.display_name,c) for c in controls]) try: g_pool.capture.frame_rate = pupil_settings_new['frame_rate'] g_pool.capture.frame_size = pupil_settings_new['frame_size'] except: logger.info("no frame rate and frame size in camera settings") for key in controls_dict: print(key) try: controls_dict[key].value = pupil_settings_new[key] except: logger.info("no key with the name '%s' in camera settings" %key) for key in pupil_settings.keys(): print(key) try: if type(pupil_settings[key]) == dict: for sec_key in pupil_settings[key].keys(): pupil_settings[key][sec_key] = pupil_settings_new[key][sec_key] else: pupil_settings[key] = pupil_settings_new[key] except: logger.info("no key with the name '%s' in pupil settings" %key) # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get( 'window_size', g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get( 'window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Selector('pupil_settings',g_pool,setter=set_pupil_settings,selection=['default','indoors','outdoors_sunny', 'outdoors_cloudy', 'vanilla'], labels=['Default', 'Indoors', 'Outdoors Sunny', 'Outdoors Cloudy', 'Vanilla'], label="Pupil settings") ) general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[ Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.capture.init_gui() g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.pupil_detector.init_gui(g_pool.sidebar) g_pool.capture_manager.init_gui() g_pool.writer = None def replace_source(source_class_name,source_settings): g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) g_pool.capture.init_gui() if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture def replace_manager(manager_class): g_pool.capture_manager.cleanup() g_pool.capture_manager = manager_class(g_pool) g_pool.capture_manager.init_gui() #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager',g_pool, setter = replace_manager, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = 'jpeg' # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject == 'eye_process.should_stop': if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye'] and g_pool.capture.online: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate) elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'): g_pool.writer = H264Writer(video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate) else: g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate) elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye{}'.format(eye_id), 'doc': eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format', 'jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process: replace_source(notification['name'],notification['args']) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get('frame') g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width): g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames and frame.jpeg_buffer: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray pupil_socket.send('frame.eye.%s'%eye_id,{ 'width': frame.width, 'height': frame.width, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) t = frame.timestamp tUnix = time() dt, ts = t - ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result, roi = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id result['unix_ts'] = tUnix #glint detection #glints = [[0,0,0,0,0,0], [0,0,0,0,0,1]] #glint_detector.glint(frame, eye_id, u_roi=g_pool.u_r, pupil=result, roi=roi) #result['glints'] = glints #g_pool.glints.put(glints) #save glint-pupil vector results #if glints[0][3]: # glint_pupil_vector = {'timestamp': glints[0][0], 'x': result['norm_pos'][0]-glints[0][3], 'y': result['norm_pos'][1]-glints[0][4], 'pupil_confidence': result['confidence'], 'glint_found': True, 'id': eye_id, 'x2': result['norm_pos'][0]-glints[1][3], 'y2': result['norm_pos'][1]-glints[1][4]} #else: # glint_pupil_vector = {'timestamp': glints[0][0], 'x': result['norm_pos'][0]-glints[0][3], 'y': result['norm_pos'][1]-glints[0][4], 'pupil_confidence': result['confidence'], 'glint_found': False, 'id': eye_id, 'x2': result['norm_pos'][0]-glints[1][3], 'y2': result['norm_pos'][1]-glints[1][4]} #g_pool.glint_pupil_vectors.put(glint_pupil_vector) # stream the result pupil_socket.send('pupil.%s'%eye_id,result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() #if frame: # glints = np.array(result['glints']) # if len(glints)>0 and glints[0][3]: # if glints[1][3]: # cygl_draw_points(glints[:,1:3], size=20,color=cygl_rgba(0.,0.,1.,.5),sharpness=1.) # elif result['confidence'] > 0.75: # cygl_draw_points(glints[:,1:3], size=20,color=cygl_rgba(0.,0.,1.,.5),sharpness=1.) f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame: if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if 'ellipse' in result: pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result['confidence'] * 0.7 draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI g_pool.gui.update() # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() # detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict() session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict() session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_gui() g_pool.pupil_detector.cleanup() glint_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() logger.info("Process shutting down.")