def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, load_object, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import init_playback_source, EndofVideoError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from batch_exporter import Batch_Exporter, Batch_Export from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from imotions_exporter import iMotions_Exporter assert VersionFormat(pyglui_version) >= VersionFormat( '1.20'), 'pyglui out of date, please upgrade to newest version' runtime_plugins = import_runtime_plugins( os.path.join(user_dir, 'plugins')) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export, System_Timelines, Audio_Playback ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, iMotions_Exporter, Offline_Calibration ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug( "Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": new_rec_dir }) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format( new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg', '.fake') video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ][0] init_playback_source(g_pool, timing='external', source_path=video_path) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get('window_size', (width, height)) window_pos = session_settings.get('window_position', window_position_default) window_name = "Pupil Player: {} - {}".format( meta_info["Recording Name"], os.path.split(rec_dir)[-1]) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( 'min_data_confidence', 0.6) g_pool.min_calibration_confidence = session_settings.get( 'min_calibration_confidence', 0.8) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data( g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer g_pool.fixations_by_frame = [ [] for x in g_pool.timestamps ] # populated by the fixation detector plugin def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time() + .8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_dir = os.path.join(g_pool.rec_dir, 'exports') export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { 'Player Software Version': str(g_pool.version), 'Data Format Version': meta_info['Data Format Version'], 'Export Date': strftime("%d.%m.%Y", localtime()), 'Export Time': strftime("%H:%M:%S", localtime()), 'Frame Index Range:': g_pool.seek_control.get_frame_index_trim_range_string(), 'Relative Time Range': g_pool.seek_control.get_rel_time_trim_range_string(), 'Absolute Time Range': g_pool.seek_control.get_abs_time_trim_range_string() } with open(os.path.join(export_dir, 'export_info.csv'), 'w') as csv: write_key_value_file(csv, export_info) notification = { 'subject': 'should_export', 'range': export_range, 'export_dir': export_dir } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.), size=(0., 0.), header_pos='headline') g_pool.user_timelines.color = RGBA(a=0.) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append(ui.Button('Reset window size', set_window_size)) general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2] + list(np.arange(1.5, 5.1, .5)), label='Interface Size')) general_settings.append( ui.Info_Text('Player Version: {}'.format(g_pool.version))) general_settings.append( ui.Info_Text('Capture Version: {}'.format( meta_info['Capture Software Version']))) general_settings.append( ui.Info_Text('Data Format Version: {}'.format( meta_info['Data Format Version']))) general_settings.append( ui.Info_Text( 'High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence.' )) general_settings.append( ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Minimum data confidence')) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100)) g_pool.export_button = ui.Thumb('export', label=chr(0xe2c5), getter=lambda: False, setter=do_export, hotkey='e', label_font='pupil_icons') g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}), ('Raw_Data_Exporter', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}), ('System_Timelines', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {}), ('Audio_Playback', {})] g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( 'rel_time_trim_section', getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label='Relative time range to export')) general_settings.insert( -1, ui.Text_Input( 'frame_idx_trim_section', getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label='Frame index range to export')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events['dt'] = get_dt() # pupil and gaze positions are added by their respective producer plugins events['pupil_positions'] = [] events['gaze_positions'] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = '' g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) # present frames at appropriate speed g_pool.seek_control.wait(events['frame'].timestamp) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings[ 'min_calibration_confidence'] = g_pool.min_calibration_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings['window_size'] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.capture.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error('Process Player crashed with trace:\n{}'.format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'}) sleep(1.0)
def init_gui(self, sidebar): self.menu = ui.Growing_Menu(label='File Capture Settings') self.menu.append( ui.Info_Text("Running Capture with '%s' as src" % self.src)) self.sidebar = sidebar self.sidebar.append(self.menu)
def close_audio_recording(self): self.menu[-2].read_only = False del self.menu[-1] self.menu.append(ui.Info_Text(NOT_REC_STR))
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id): # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # general imports import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D} # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (600,31+ 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.process = 'eye{}'.format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0] g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,g_pool.capture.frame_size ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id))) if VersionFormat(session_settings.get("version", '0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get( 'display_mode', 'camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager',{})) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__:c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings) if eye_id == 0: cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"] else: cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"] # Initialize capture default_settings = ('UVC_Source',{ 'preferred_names' : cap_src, 'frame_size': (640,480), 'frame_rate': 90 }) capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__:c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector( g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get( 'window_size', g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get( 'window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[0.5, 0.75, 1., 1.5, 2.], label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[ Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.capture.init_gui() g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.pupil_detector.init_gui(g_pool.sidebar) g_pool.capture_manager.init_gui() g_pool.writer = None def replace_source(source_class_name,source_settings): g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) g_pool.capture.init_gui() if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture def replace_manager(manager_class): g_pool.capture_manager.cleanup() g_pool.capture_manager = manager_class(g_pool) g_pool.capture_manager.init_gui() #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager',g_pool, setter = replace_manager, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = 'jpeg' # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject == 'eye_process.should_stop': if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye'] and g_pool.capture.online: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate) elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'): g_pool.writer = H264Writer(video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate) else: g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate) elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye{}'.format(eye_id), 'doc': eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format', 'jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process: replace_source(notification['name'],notification['args']) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get('frame') g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width): g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames and frame.jpeg_buffer: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray pupil_socket.send('frame.eye.%s'%eye_id,{ 'width': frame.width, 'height': frame.width, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s'%eye_id,result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame: if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if 'ellipse' in result: pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result['confidence'] * 0.7 draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI g_pool.gui.update() # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() # detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict() session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict() session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_gui() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() logger.info("Process shutting down.")
def demo(): global quit quit = False # Callback functions def on_resize(window, w, h): h = max(h, 1) w = max(w, 1) hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize( window)[0] w, h = w * hdpi_factor, h * hdpi_factor gui.update_window(w, h) active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) # norm_size = normalize((w,h),glfwGetWindowSize(window)) # fb_size = denormalize(norm_size,glfwGetFramebufferSize(window)) adjust_gl_view(w, h, window) glfwMakeContextCurrent(active_window) def on_iconify(window, iconfied): pass def on_key(window, key, scancode, action, mods): gui.update_key(key, scancode, action, mods) if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: on_close(window) def on_char(window, char): gui.update_char(char) def on_button(window, button, action, mods): gui.update_button(button, action, mods) # pos = normalize(pos,glfwGetWindowSize(window)) # pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor gui.update_mouse(x, y) def on_scroll(window, x, y): gui.update_scroll(x, y) def on_close(window): global quit quit = True logger.info('Process closing from window') # get glfw started glfwInit() window = glfwCreateWindow(width, height, "pyglui demo", None, None) if not window: exit() glfwSetWindowPos(window, 0, 0) # Register callbacks window glfwSetWindowSizeCallback(window, on_resize) glfwSetWindowCloseCallback(window, on_close) glfwSetWindowIconifyCallback(window, on_iconify) glfwSetKeyCallback(window, on_key) glfwSetCharCallback(window, on_char) glfwSetMouseButtonCallback(window, on_button) glfwSetCursorPosCallback(window, on_pos) glfwSetScrollCallback(window, on_scroll) glfwMakeContextCurrent(window) init() basic_gl_setup() class Temp(object): """Temp class to make objects""" def __init__(self): pass foo = Temp() foo.bar = 34 foo.bur = 4 foo.mytext = [203, 12] foo.myswitch = 10 foo.select = 'Tiger' foo.record = False foo.calibrate = False foo.stream = True foo.test = False d = {} d['one'] = 1 def print_hello(): foo.select = 'Cougar' gui.scale += .1 print 'hello' # m.configuration = sidebar.configuration def printer(val): print 'setting to :', val print "pyglui version: %s" % (ui.__version__) gui = ui.UI() gui.scale = 1.0 sidebar = ui.Scrolling_Menu("MySideBar", pos=(-300, 0), size=(0, 0), header_pos='left') sm = ui.Growing_Menu("SubMenu", pos=(0, 0), size=(0, 100)) sm.append(ui.Slider("bar", foo)) sm.append(ui.Text_Input('mytext', foo, setter=printer)) ssm = ui.Growing_Menu("SubSubMenu", pos=(0, 0), size=(0, 100)) ssm.append(ui.Slider("bar", foo)) ssm.append(ui.Text_Input('mytext', foo, setter=printer)) sm.append(ssm) sidebar.append(sm) sm.append( ui.Selector('select', foo, selection=['Tiger', 'Lion', 'Cougar', 'Hyena'])) sm.append(ui.Button("Say Hi!", print_hello)) gui.append(sidebar) m = ui.Scrolling_Menu("MyMenu", pos=(250, 30), size=(300, 500), header_pos='top') m.append( ui.Info_Text( "This is my multiline info text. I wonder if multilines break as designed... How does it look? Info Text with long label text to test multiline break handling." )) m.append( ui.Selector('select', foo, selection=['Tiger', 'Lion', 'Cougar', 'Hyena'], setter=printer)) m.append( ui.Slider( "bur", foo, step=50, min=1, max=1005, label="Slider label with long label text to test overflow handling" )) m.append(ui.Button("Say Hi!", print_hello)) m.append( ui.Switch("myswitch", foo, on_val=1000, off_val=10, label="Switch Me")) sm = ui.Growing_Menu("SubMenu", pos=(0, 0), size=(0, 100)) sm.append(ui.Slider("bar", foo)) sm.append(ui.Text_Input('mytext', foo)) m.append(sm) m.append(ui.Button("Say Hi!", print_hello)) rightbar = ui.Stretching_Menu('Right Bar', (0, 100), (150, -100)) rightbar.append(ui.Thumb("record", foo, label="Record")) rightbar.append(ui.Thumb("calibrate", foo, label="Calibrate")) rightbar.append(ui.Thumb("stream", foo, label="Stream")) rightbar.append(ui.Thumb("test", foo, label="Test")) gui.append(rightbar) gui.append(m) import os import psutil pid = os.getpid() ps = psutil.Process(pid) ts = time.time() from pyglui import graph print graph.__version__ cpu_g = graph.Line_Graph() cpu_g.pos = (50, 100) cpu_g.update_fn = ps.get_cpu_percent cpu_g.update_rate = 5 cpu_g.label = 'CPU %0.1f' fps_g = graph.Line_Graph() fps_g.pos = (50, 100) fps_g.update_rate = 5 fps_g.label = "%0.0f FPS" fps_g.color[:] = .1, .1, .8, .9 st_graph = graph.Averaged_Value() st_graph.pos = (200, 200) st_graph.update_rate = 5 st_graph.label = "Slider Value: %0.0f" st_graph.color[:] = 1., 0., .6, .9 on_resize(window, *glfwGetWindowSize(window)) while not quit: dt, ts = time.time() - ts, time.time() clear_gl_screen() cpu_g.update() cpu_g.draw() fps_g.add(1. / dt) fps_g.draw() st_graph.add(foo.bur) st_graph.draw() gui.update() glfwSwapBuffers(window) glfwPollEvents() glfwDestroyWindow(window) glfwTerminate() logger.debug("Process done")
def eye(g_pool, cap_src, cap_size, rx_from_world, eye_id=0): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ # modify the root logger for this process logger = logging.getLogger() # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye%s.log' % eye_id), mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'Eye' + str(eye_id) + ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'EYE' + str(eye_id) + ' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # create logger for the context of this function logger = logging.getLogger(__name__) # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize( window)[0] w, h = w * hdpi_factor, h * hdpi_factor g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) # for p in g_pool.plugins: # p.on_window_resize(window,w,h) glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == GLFW_RELEASE and u_r.active_edit_pt: u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == GLFW_PRESS: pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40, u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if u_r.active_edit_pt: pos = normalize((x, y), glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) u_r.move_vertex(u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 30, timebase=g_pool.timebase) # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.window_size = session_settings.get('window_size', 1.) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below." } # g_pool.draw_pupil = session_settings.get('draw_pupil',True) u_r = UIRoi(frame.img.shape) u_r.set(session_settings.get('roi', u_r.get())) writer = None pupil_detector = Canny_Detector(g_pool) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def get_scale(): return g_pool.gui.scale def set_display_mode_info(val): # set info text here and append to the general settings menu # 'camera_image','roi','algorithm','cpu_save' g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] width, height = session_settings.get('window_size', (frame.width, frame.height)) window_pos = session_settings.get('window_position', (0, 0)) # not yet using this one. # Initialize glfw glfwInit() if g_pool.binocular: title = "Binocular eye %s" % eye_id else: title = 'Eye' main_window = glfwCreateWindow(width, height, title, None, None) glfwMakeContextCurrent(main_window) cygl_init() # Register callbacks main_window glfwSetWindowSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img.shape) update_named_texture(g_pool.image_tex, frame.img) # refresh speed settings glfwSwapInterval(0) glfwSetWindowPos(main_window, 800, 300 * eye_id) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') g_pool.sidebar.configuration = session_settings.get( 'side_bar_config', {'collapsed': True}) general_settings = ui.Growing_Menu('General') general_settings.configuration = session_settings.get( 'general_menu_config', {}) general_settings.append( ui.Slider('scale', setter=set_scale, getter=get_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize(main_window, frame.width, frame.height))) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append( ui.Hot_Key("quit", setter=on_close, getter=lambda: True, label="X", hotkey=GLFW_KEY_ESCAPE)) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.menu.configuration = session_settings.get( 'capture_menu_config', {'collapsed': True}) # let detector add its GUI pupil_detector.init_gui(g_pool.sidebar) #set the last saved window size on_resize(main_window, *glfwGetWindowSize(main_window)) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.get_cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if rx_from_world.poll(): command = rx_from_world.recv() if command is not None: record_path = command logger.info("Will save eye video to: %s" % record_path) video_path = os.path.join(record_path, "eye%s.mkv" % eye_id) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy" % eye_id) writer = cv2.VideoWriter( video_path, cv2.cv.CV_FOURCC(*'DIVX'), float(cap.frame_rate), (frame.img.shape[1], frame.img.shape[0])) timestamps = [] else: logger.info("Done recording.") writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write(frame.img) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect( frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': update_named_texture(g_pool.image_tex, frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): update_named_texture(g_pool.image_tex, frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) draw_named_texture(g_pool.image_tex) # switch to work in pixel space make_coord_system_pixel_based((frame.height, frame.width, 3), g_pool.flip) if result['confidence'] > 0: if result.has_key('axes'): pts = cv2.ellipse2Poly( (int(result['center'][0]), int(result['center'][1])), (int(result['axes'][0] / 2), int(result['axes'][1] / 2)), int(result['angle']), 0, 360, 15) cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5)) cygl_draw_points([result['center']], size=20, color=cygl_rgba(1., 0., 0., .5), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': u_r.draw(g_pool.gui.scale) #update screen glfwSwapBuffers(main_window) glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['side_bar_config'] = g_pool.sidebar.configuration session_settings['capture_menu_config'] = g_pool.capture.menu.configuration session_settings['general_menu_config'] = general_settings.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings.close() pupil_detector.cleanup() cap.close() glfwDestroyWindow(main_window) glfwTerminate() #flushing queue in case world process did not exit gracefully while not g_pool.pupil_queue.empty(): g_pool.pupil_queue.get() g_pool.pupil_queue.close() logger.debug("Process done")
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {"subject": "set_detection_mapping_mode", "mode": new_mode} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( "1.24"), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_routines import ( calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin, ) from fixation_detector import Fixation_Detector from eye_movement import Eye_Movement_Detector_Real_Time from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins")) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Eye_Movement_Detector_Real_Time, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ([ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes) plugins = (system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ ("UVC_Source", default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), ("Screen_Marker_Calibration", {}), ("Recorder", {}), ("Pupil_Remote", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for plugin in g_pool.plugins: if plugin.on_drop(paths): break tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world")) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) g_pool.detection_mapping_mode = session_settings.get( "detection_mapping_mode", "3d") g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode) def handle_notifications(noti): subject = noti["subject"] if subject == "set_detection_mapping_mode": if noti["mode"] == "2d": if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add( g_pool.plugin_by_name["Dummy_Gaze_Mapper"]) g_pool.detection_mapping_mode = noti["mode"] elif subject == "start_plugin": try: g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_detection_mapping_mode", "mode": g_pool.detection_mapping_mode, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) elif subject == "world_process.adapt_window_size": set_window_size() width, height = session_settings.get("window_size", (1280 + icon_bar_width, 720)) # window and gl setup glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "world_process.should_start", "delay": 2.0 }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.6, 0.8, 1.0, 1.2, 1.4], label="Interface size", )) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) on_resize(main_window, f_width, f_height) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector("audio_mode", audio, selection=audio.audio_modes)) general_settings.append( ui.Selector( "detection_mapping_mode", g_pool, label="detection & mapping mode", setter=set_detection_mapping_mode, selection=["disabled", "2d", "3d"], )) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, )) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, )) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version))) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins glfw.glfwPollEvents() if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.glfwSwapBuffers(main_window) glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings[ "detection_mapping_mode"] = g_pool.detection_mapping_mode session_settings["audio_mode"] = audio.audio_mode session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def init_ui(self): self.add_menu() self.menu.label = "Recorder" self.menu_icon.order = 0.29 self.menu.append( ui.Info_Text( 'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.' ) ) self.menu.append( ui.Info_Text( 'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.' ) ) self.menu.append( ui.Text_Input( "rec_root_dir", self, setter=self.set_rec_root_dir, label="Path to recordings", ) ) self.menu.append( ui.Text_Input( "session_name", self, setter=self.set_session_name, label="Recording session name", ) ) self.menu.append( ui.Switch( "show_info_menu", self, on_val=True, off_val=False, label="Request additional user info", ) ) self.menu.append( ui.Selector( "raw_jpeg", self, selection=[True, False], labels=["bigger file, less CPU", "smaller file, more CPU"], label="Compression", ) ) self.menu.append( ui.Info_Text( "Recording the raw eye video is optional. We use it for debugging." ) ) self.menu.append( ui.Switch( "record_eye", self, on_val=True, off_val=False, label="Record eye" ) ) self.button = ui.Thumb( "running", self, setter=self.toggle, label="R", hotkey="r" ) self.button.on_color[:] = (1, 0.0, 0.0, 0.8) self.g_pool.quickbar.insert(2, self.button) self.low_disk_space_thumb = ui.Thumb( "low_disk_warn", label="!", getter=lambda: True, setter=lambda x: None ) self.low_disk_space_thumb.on_color[:] = (1, 0.0, 0.0, 0.8) self.low_disk_space_thumb.status_text = "Low disk space"
def init_ui(self): self.add_menu() self.menu.label = "IMU Timeline" self.menu.append(ui.Info_Text("Visualize IMU data and export to .csv file")) self.menu.append( ui.Info_Text( "This plugin visualizes accelerometer, gyroscope and " " orientation data from Pupil Invisible recordings. Results are " " exported in 'imu_timeline.csv' " ) ) self.menu.append( ui.Info_Text( "Orientation is estimated using Madgwick's algorithm. " " Madgwick implements a beta value which is related with the " " error of the gyroscope. Increasing the beta leads to faster " " corrections but with more sensitivity to lateral accelerations. " " Read more about Madgwick's algorithm here: " " https://www.x-io.co.uk/res/doc/madgwick_internal_report.pdf " ) ) def set_gyro_error(new_value): self.gyro_error = new_value self.notify_all({"subject": "madgwick_fusion.should_fuse", "delay": 0.3}) self.menu.append( ui.Switch( "should_draw_raw", self, label="View raw timeline", setter=self.on_draw_raw_toggled, ) ) self.menu.append( ui.Switch( "should_draw_orientation", self, label="View orientation timeline", setter=self.on_draw_orientation_toggled, ) ) self.menu.append( ui.Slider( "gyro_error", self, min=1, step=0.1, max=100, label="Madgwick's beta", setter=set_gyro_error, ) ) if self.should_draw_raw: self.append_timeline_raw() if self.should_draw_orientation: self.append_timeline_orientation() if self.data_orient.shape[0] == 0: # Start fusion after setting up timelines self._fuse()
def init_ui(self): desc_text = ui.Info_Text(self._choreography_description_text()) self.__ui_selector_choreography = ui.Selector( "selected_choreography_class", self, label="Choreography", selection_getter=self.__choreography_selection_getter, ) self.__ui_selector_gazer = ui.Selector( "selected_gazer_class", self, label="Gaze Mapping", labels=[g.label for g in self.user_selectable_gazer_classes()], selection=self.user_selectable_gazer_classes(), ) self.__ui_gazer_description_text = ui.Info_Text("") self._update_gazer_description_ui_text() best_practices_text = ui.Info_Text( "Read more about best practices at docs.pupil-labs.com") custom_ui_elements = self._init_custom_menu_ui_elements() super().init_ui() self.add_menu() self.menu.label = self.label self.menu_icon.order = self.order self.menu_icon.tooltip = "Calibration" # Construct menu UI self.menu.append(self.__ui_selector_choreography) self.menu.append(desc_text) if len(custom_ui_elements) > 0: self.menu.append(ui.Separator()) for ui_elem in custom_ui_elements: self.menu.append(ui_elem) self.menu.append(ui.Separator()) else: self.menu.append(ui.Separator()) self.menu.append(self.__ui_selector_gazer) self.menu.append(self.__ui_gazer_description_text) self.menu.append(best_practices_text) if self.shows_action_buttons: def calibration_setter(should_be_on): self.__signal_should_toggle_processing( should_be_on=should_be_on, mode=ChoreographyMode.CALIBRATION) def validation_setter(should_be_on): self.__signal_should_toggle_processing( should_be_on=should_be_on, mode=ChoreographyMode.VALIDATION) self.__ui_button_calibration = ui.Thumb( "is_active", self, label="C", hotkey="c", setter=calibration_setter, on_color=self._THUMBNAIL_COLOR_ON, ) self.__ui_button_validation = ui.Thumb( "is_active", self, label="T", hotkey="t", setter=validation_setter, on_color=self._THUMBNAIL_COLOR_ON, ) self.__toggle_mode_button_visibility( is_visible=True, mode=ChoreographyMode.CALIBRATION) self.__toggle_mode_button_visibility( is_visible=True, mode=ChoreographyMode.VALIDATION)
def init_gui(self): from pyglui import ui ui_elements = [] #lets define some helper functions: def gui_load_defaults(): for c in self.uvc_capture.controls: try: c.value = c.def_val except: pass def gui_update_from_device(): for c in self.uvc_capture.controls: c.refresh() ui_elements.append(ui.Info_Text('%s Controls'%self.uvc_capture.name)) sensor_control = ui.Growing_Menu(label='Sensor Settings') sensor_control.append(ui.Info_Text("Do not change these during calibration or recording!")) sensor_control.collapsed=False image_processing = ui.Growing_Menu(label='Image Post Processing') image_processing.collapsed=True sensor_control.append(ui.Selector( 'frame_size',self, setter=self.set_frame_size, selection=self.uvc_capture.frame_sizes, label='Resolution' )) sensor_control.append(ui.Selector('frame_rate',self, selection=self.uvc_capture.frame_rates,label='Frame rate' ) ) for control in self.uvc_capture.controls: c = None ctl_name = control.display_name #now we add controls if control.d_type == bool : c = ui.Switch('value',control,label=ctl_name, on_val=control.max_val, off_val=control.min_val) elif control.d_type == int: c = ui.Slider('value',control,label=ctl_name,min=control.min_val,max=control.max_val,step=control.step) elif type(control.d_type) == dict: selection = [value for name,value in control.d_type.iteritems()] labels = [name for name,value in control.d_type.iteritems()] c = ui.Selector('value',control, label = ctl_name, selection=selection,labels = labels) else: pass # if control['disabled']: # c.read_only = True # if ctl_name == 'Exposure, Auto Priority': # # the controll should always be off. we set it to 0 on init (see above) # c.read_only = True if c is not None: if control.unit == 'processing_unit': image_processing.append(c) else: sensor_control.append(c) ui_elements.append(sensor_control) if image_processing.elements: ui_elements.append(image_processing) ui_elements.append(ui.Button("refresh",gui_update_from_device)) ui_elements.append(ui.Button("load defaults",gui_load_defaults)) self.g_pool.capture_source_menu.extend(ui_elements)
def init_ui(self): self.add_menu() self.menu.label = 'Fixation Detector' def set_max_dispersion(new_value): self.max_dispersion = new_value self.notify_all({ 'subject': 'fixation_detector.should_recalculate', 'delay': 1. }) def set_min_duration(new_value): self.min_duration = min(new_value, self.max_duration) self.notify_all({ 'subject': 'fixation_detector.should_recalculate', 'delay': 1. }) def set_max_duration(new_value): self.max_duration = max(new_value, self.min_duration) self.notify_all({ 'subject': 'fixation_detector.should_recalculate', 'delay': 1. }) def jump_next_fixation(_): cur_idx = self.last_frame_idx all_idc = [f['mid_frame_index'] for f in self.g_pool.fixations] if not all_idc: logger.warning('No fixations available') return # wrap-around index tar_fix = bisect_right(all_idc, cur_idx) % len(all_idc) self.notify_all({ 'subject': 'seek_control.should_seek', 'index': int(self.g_pool.fixations[tar_fix]['mid_frame_index']) }) def jump_prev_fixation(_): cur_idx = self.last_frame_idx all_idc = [f['mid_frame_index'] for f in self.g_pool.fixations] if not all_idc: logger.warning('No fixations available') return # wrap-around index tar_fix = (bisect_left(all_idc, cur_idx) - 1) % len(all_idc) self.notify_all({ 'subject': 'seek_control.should_seek', 'index': int(self.g_pool.fixations[tar_fix]['mid_frame_index']) }) for help_block in self.__doc__.split('\n\n'): help_str = help_block.replace('\n', ' ').replace(' ', '').strip() self.menu.append(ui.Info_Text(help_str)) self.menu.append( ui.Info_Text( "Press the export button or type 'e' to start the export.")) self.menu.append( ui.Slider('max_dispersion', self, min=0.01, step=0.1, max=5., label='Maximum Dispersion [degrees]', setter=set_max_dispersion)) self.menu.append( ui.Slider('min_duration', self, min=10, step=10, max=4000, label='Minimum Duration [milliseconds]', setter=set_min_duration)) self.menu.append( ui.Slider('max_duration', self, min=10, step=10, max=4000, label='Maximum Duration [milliseconds]', setter=set_max_duration)) self.menu.append( ui.Text_Input('status', self, label='Detection progress:', setter=lambda x: None)) self.menu.append( ui.Switch('show_fixations', self, label='Show fixations')) self.current_fixation_details = ui.Info_Text('') self.menu.append(self.current_fixation_details) self.next_fix_button = ui.Thumb('jump_next_fixation', setter=jump_next_fixation, getter=lambda: False, label=chr(0xe044), hotkey='f', label_font='pupil_icons') self.next_fix_button.status_text = 'Next Fixation' self.g_pool.quickbar.append(self.next_fix_button) self.prev_fix_button = ui.Thumb('jump_prev_fixation', setter=jump_prev_fixation, getter=lambda: False, label=chr(0xe045), hotkey='F', label_font='pupil_icons') self.prev_fix_button.status_text = 'Previous Fixation' self.g_pool.quickbar.append(self.prev_fix_button)
def update_gui_markers(self): def close(): self.alive = False def set_min_marker_perimeter(val): self.min_marker_perimeter = val self.notify_all({ 'subject': 'min_marker_perimeter_changed', 'delay': 1 }) def set_invert_image(val): self.invert_image = val self.invalidate_marker_cache() self.invalidate_surface_caches() self.menu.elements[:] = [] self.menu.append(ui.Button('Close', close)) self.menu.append( ui.Switch('invert_image', self, setter=set_invert_image, label='Use inverted markers')) self.menu.append( ui.Slider('min_marker_perimeter', self, min=20, max=500, step=1, setter=set_min_marker_perimeter)) self.menu.append( ui.Info_Text( 'The offline surface tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.' )) self.menu.append( ui.Info_Text( "Press the export button or type 'e' to start the export.")) self.menu.append( ui.Selector('mode', self, label='Mode', selection=[ "Show Markers and Surfaces", "Show marker IDs", "Show Heatmaps", "Show Metrics" ])) self.menu.append( ui.Info_Text( 'To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.' )) self.menu.append( ui.Button("(Re)-calculate gaze distributions", self.recalculate)) self.menu.append(ui.Button("Add surface", lambda: self.add_surface())) for s in self.surfaces: idx = self.surfaces.index(s) s_menu = ui.Growing_Menu("Surface {}".format(idx)) s_menu.collapsed = True s_menu.append(ui.Text_Input('name', s)) s_menu.append(ui.Text_Input('x', s.real_world_size, label='X size')) s_menu.append(ui.Text_Input('y', s.real_world_size, label='Y size')) s_menu.append(ui.Button('Open Debug Window', s.open_close_window)) #closure to encapsulate idx def make_remove_s(i): return lambda: self.remove_surface(i) remove_s = make_remove_s(idx) s_menu.append(ui.Button('remove', remove_s)) self.menu.append(s_menu)
def update_menu(self): del self.menu[:] from pyglui import ui ui_elements = [] # lets define some helper functions: def gui_load_defaults(): for c in self.uvc_capture.controls: try: c.value = c.def_val except: pass def gui_update_from_device(): for c in self.uvc_capture.controls: c.refresh() def set_frame_size(new_size): self.frame_size = new_size def set_frame_rate(new_rate): self.frame_rate = new_rate self.update_menu() if self.uvc_capture is None: ui_elements.append(ui.Info_Text("Capture initialization failed.")) self.menu.extend(ui_elements) return ui_elements.append(ui.Info_Text("{} Controls".format(self.name))) sensor_control = ui.Growing_Menu(label="Sensor Settings") sensor_control.append( ui.Info_Text( "Do not change these during calibration or recording!")) sensor_control.collapsed = False image_processing = ui.Growing_Menu(label="Image Post Processing") image_processing.collapsed = True sensor_control.append( ui.Selector( "frame_size", self, setter=set_frame_size, selection=self.uvc_capture.frame_sizes, label="Resolution", )) def frame_rate_getter(): return ( self.uvc_capture.frame_rates, [str(fr) for fr in self.uvc_capture.frame_rates], ) sensor_control.append( ui.Selector( "frame_rate", self, selection_getter=frame_rate_getter, setter=set_frame_rate, label="Frame rate", )) if ("Pupil Cam2" in self.uvc_capture.name or "Pupil Cam3" in self.uvc_capture.name): special_settings = {200: 28, 180: 31} def set_exposure_mode(exposure_mode): self.exposure_mode = exposure_mode if self.exposure_mode == "auto": self.preferred_exposure_time = Exposure_Time( max_ET=special_settings.get(self.frame_rate, 32), frame_rate=self.frame_rate, mode=self.exposure_mode, ) else: self.preferred_exposure_time = None logger.info( "Exposure mode for camera {0} is now set to {1} mode". format(self.uvc_capture.name, exposure_mode)) self.update_menu() def exposure_mode_getter(): return ["manual", "auto"], ["manual mode", "auto mode"] sensor_control.append( ui.Selector( "exposure_mode", self, setter=set_exposure_mode, selection_getter=exposure_mode_getter, selection=self.exposure_mode, label="Exposure Mode", )) sensor_control.append( ui.Slider( "exposure_time", self, label="Absolute Exposure Time", min=1, max=special_settings.get(self.frame_rate, 32), step=1, )) if self.exposure_mode == "auto": sensor_control[-1].read_only = True if "Pupil Cam" in self.uvc_capture.name: blacklist = [ "Auto Focus", "Absolute Focus", "Absolute Iris ", "Scanning Mode ", "Zoom absolute control", "Pan control", "Tilt control", "Roll absolute control", "Privacy Shutter control", ] else: blacklist = [] if ("Pupil Cam2" in self.uvc_capture.name or "Pupil Cam3" in self.uvc_capture.name): blacklist += [ "Auto Exposure Mode", "Auto Exposure Priority", "Absolute Exposure Time", ] for control in self.uvc_capture.controls: c = None ctl_name = control.display_name if ctl_name in blacklist: continue # now we add controls if control.d_type == bool: c = ui.Switch( "value", control, label=ctl_name, on_val=control.max_val, off_val=control.min_val, ) elif control.d_type == int: c = ui.Slider( "value", control, label=ctl_name, min=control.min_val, max=control.max_val, step=control.step, ) elif type(control.d_type) == dict: selection = [value for name, value in control.d_type.items()] labels = [name for name, value in control.d_type.items()] c = ui.Selector("value", control, label=ctl_name, selection=selection, labels=labels) else: pass # if control['disabled']: # c.read_only = True # if ctl_name == 'Exposure, Auto Priority': # # the controll should always be off. we set it to 0 on init (see above) # c.read_only = True if c is not None: if control.unit == "processing_unit": image_processing.append(c) else: sensor_control.append(c) ui_elements.append(sensor_control) if image_processing.elements: ui_elements.append(image_processing) ui_elements.append(ui.Button("refresh", gui_update_from_device)) if "Pupil Cam2" in self.uvc_capture.name: def set_check_stripes(enable_stripe_checks): self.enable_stripe_checks = enable_stripe_checks if self.enable_stripe_checks: self.stripe_detector = Check_Frame_Stripes() logger.info("Check Stripes for camera {} is now on".format( self.uvc_capture.name)) else: self.stripe_detector = None logger.info( "Check Stripes for camera {} is now off".format( self.uvc_capture.name)) ui_elements.append( ui.Switch( "enable_stripe_checks", self, setter=set_check_stripes, label="Check Stripes", )) self.menu.extend(ui_elements)
def add_controls_to_menu(self, menu, controls): from pyglui import ui # closure factory def make_value_change_fn(ctrl_id): def initiate_value_change(val): logger.debug('{}: {} >> {}'.format(self.sensor, ctrl_id, val)) self.sensor.set_control_value(ctrl_id, val) return initiate_value_change for ctrl_id, ctrl_dict in controls: try: dtype = ctrl_dict['dtype'] ctrl_ui = None if dtype == "string": ctrl_ui = ui.Text_Input( 'value', ctrl_dict, label=ctrl_dict['caption'], setter=make_value_change_fn(ctrl_id)) elif dtype == "integer" or dtype == "float": convert_fn = int if dtype == "integer" else float ctrl_ui = ui.Slider( 'value', ctrl_dict, label=ctrl_dict['caption'], min=convert_fn(ctrl_dict.get('min', 0)), max=convert_fn(ctrl_dict.get('max', 100)), step=convert_fn(ctrl_dict.get('res', 0.)), setter=make_value_change_fn(ctrl_id)) elif dtype == "bool": ctrl_ui = ui.Switch('value', ctrl_dict, label=ctrl_dict['caption'], on_val=ctrl_dict.get('max', True), off_val=ctrl_dict.get('min', False), setter=make_value_change_fn(ctrl_id)) elif dtype == "strmapping" or dtype == "intmapping": desc_list = ctrl_dict['map'] labels = [desc['caption'] for desc in desc_list] selection = [desc['value'] for desc in desc_list] ctrl_ui = ui.Selector('value', ctrl_dict, label=ctrl_dict['caption'], labels=labels, selection=selection, setter=make_value_change_fn(ctrl_id)) if ctrl_ui: ctrl_ui.read_only = ctrl_dict.get('readonly', False) self.control_id_ui_mapping[ctrl_id] = ctrl_ui menu.append(ctrl_ui) else: logger.error('Did not generate UI for {}'.format(ctrl_id)) except: logger.error('Exception for control:\n{}'.format(ctrl_dict)) import traceback as tb tb.print_exc() if len(menu) == 0: menu.append(ui.Info_Text("No {} settings found".format( menu.label))) return menu
def init_ui(self): self.add_menu() self.menu.label = "Static Image Source" text = ui.Info_Text("Fake capture source streaming test images.") self.menu.append(text)
def init_gui(self): from pyglui import ui ui_elements = [] ui_elements.append(ui.Info_Text('Remote Pupil Mobile sources')) def host_selection_list(): devices = { s['host_uuid']: s['host_name'] # removes duplicates for s in self.network.sensors.values() } devices = [pair for pair in devices.items()] # create tuples # split tuples into 2 lists return zip(*(devices or [(None, 'No hosts found')])) def view_host(host_uuid): if self.selected_host != host_uuid: self.selected_host = host_uuid self.re_build_ndsi_menu() host_sel, host_sel_labels = host_selection_list() ui_elements.append( ui.Selector('selected_host', self, selection=host_sel, labels=host_sel_labels, setter=view_host, label='Remote host')) self.g_pool.capture_selector_menu.extend(ui_elements) if not self.selected_host: return ui_elements = [] host_menu = ui.Growing_Menu('Remote Host Information') ui_elements.append(host_menu) def source_selection_list(): default = (None, 'Select to activate') # self.poll_events() sources = [default] + [(s['sensor_uuid'], s['sensor_name']) for s in self.network.sensors.values() if (s['sensor_type'] == 'video' and s['host_uuid'] == self.selected_host)] return zip(*sources) def activate(source_uid): if not source_uid: return settings = { 'frame_size': self.g_pool.capture.frame_size, 'frame_rate': self.g_pool.capture.frame_rate, 'source_id': source_uid } self.activate_source(settings) src_sel, src_sel_labels = source_selection_list() host_menu.append( ui.Selector('selected_source', selection=src_sel, labels=src_sel_labels, getter=lambda: None, setter=activate, label='Source')) self.g_pool.capture_selector_menu.extend(ui_elements)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils # capture from video_capture import File_Source, EndofVideoFileError, FileSeekError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # monitoring import psutil # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_bar import Seek_Bar from trim_marks import Trim_Marks from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector # from manual_gaze_correction import Manual_Gaze_Correction from batch_exporter import Batch_Exporter from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration assert pyglui_version >= '1.7' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url # Initialize capture cap = File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index() + 1) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 1) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) g_pool.main_menu.append(ui.Info_Text('Open plugins')) selector_label = "Select to load" def append_selector(label, plugins): plugins.sort(key=lambda p: p.__name__) plugin_labels = [p.__name__.replace('_', ' ') for p in plugins] g_pool.main_menu.append(ui.Selector(label, selection=[selector_label] + plugins, labels=[selector_label] + plugin_labels, setter=open_plugin, getter=lambda: selector_label)) base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base] base_labels = ['Visualizer:', 'Analyser:', 'Data Source:'] launchable = user_launchable_plugins.copy() for base_class, label in zip(base_plugins, base_labels): member_plugins = [] for p in user_launchable_plugins: if issubclass(p, base_class): member_plugins.append(p) launchable.remove(p) append_selector(label, member_plugins) # launchable only contains plugins that could not be assigned to any of the above categories append_selector('Other', launchable) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=glfw.GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=glfw.GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=glfw.GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_ndarray(frame.bgr) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() unused_elements = g_pool.gui.update() for b in unused_elements.buttons: button, action, mods = b pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) # present frames at appropriate speed cap.wait(frame) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'})
def init_ui(self): self.add_menu() self.menu.label = 'Fixation Detector' def set_max_dispersion(new_value): self.max_dispersion = new_value self.notify_all({ 'subject': 'fixation_detector.should_recalculate', 'delay': 1. }) def set_min_duration(new_value): self.min_duration = min(new_value, self.max_duration) self.notify_all({ 'subject': 'fixation_detector.should_recalculate', 'delay': 1. }) def set_max_duration(new_value): self.max_duration = max(new_value, self.min_duration) self.notify_all({ 'subject': 'fixation_detector.should_recalculate', 'delay': 1. }) def jump_next_fixation(_): ts = self.last_frame_ts for f in self.g_pool.fixations: if f['timestamp'] > ts: self.g_pool.capture.seek_to_frame(f['mid_frame_index']) self.g_pool.new_seek = True return logger.error('No further fixation available') for help_block in self.__doc__.split('\n\n'): help_str = help_block.replace('\n', ' ').replace(' ', '').strip() self.menu.append(ui.Info_Text(help_str)) self.menu.append( ui.Info_Text( "Press the export button or type 'e' to start the export.")) self.menu.append( ui.Slider('max_dispersion', self, min=0.01, step=0.1, max=5., label='Maximum Dispersion [degrees]', setter=set_max_dispersion)) self.menu.append( ui.Slider('min_duration', self, min=10, step=10, max=1500, label='Minimum Duration [milliseconds]', setter=set_min_duration)) self.menu.append( ui.Slider('max_duration', self, min=10, step=10, max=1500, label='Maximum Duration [milliseconds]', setter=set_max_duration)) self.menu.append( ui.Text_Input('status', self, label='Detection progress:', setter=lambda x: None)) self.menu.append( ui.Switch('show_fixations', self, label='Show fixations')) self.current_fixation_details = ui.Info_Text('') self.menu.append(self.current_fixation_details) self.add_button = ui.Thumb('jump_next_fixation', setter=jump_next_fixation, getter=lambda: False, label=chr(0xe044), hotkey='f', label_font='pupil_icons') self.add_button.status_text = 'Next Fixation' self.g_pool.quickbar.append(self.add_button)
def init_gui(self,sidebar): def gui_init_cam(d): self.re_init(d.symbolicName, (self.width, self.height), self.preferred_fps) def gui_get_cam(): return self.name def gui_get_frame_size(): return self.frame_size def gui_set_frame_size(new_size): self.frame_size = new_size def gui_get_frame_rate(): return self.frame_rate def gui_set_frame_rate(new_fps): self.frame_rate = new_fps #create the menu entry self.menu = ui.Growing_Menu(label='Camera Settings') #cams = Camera_List() #cam_names = [str(c.name) for c in cams] #cam_devices = [c.device for c in cams] #self.menu.append(ui.Selector('device',self,selection=cam_devices,labels=cam_names,label='Capture Device', getter=gui_get_cam, setter=gui_init_cam)) self.menu.append(ui.Info_Text("Device: " + self.name)) #hardware_ts_switch = ui.Switch('use_hw_ts',self,label='use hardware timestamps') #hardware_ts_switch.read_only = True #self.menu.append(hardware_ts_switch) #self.menu.append(ui.Selector('frame_size', selection=self.available_frame_sizes, label='Frame Size', getter=gui_get_frame_size, setter=gui_set_frame_size)) self.menu.append(ui.Info_Text("Resolution: {0} x {1} pixels".format(self.actual_width, self.actual_height))) self.menu.append(ui.Selector('frame_rate', selection=self.available_frame_rates, label='Frame Rate', getter=gui_get_frame_rate, setter=gui_set_frame_rate)) # for control in self.controls: # c = None # ctl_name = control['name'] # # we use closures as setters and getters for each control element # def make_setter(control): # def fn(val): # self.capture.set_control(control['id'],val) # control['value'] = self.capture.get_control(control['id']) # return fn # def make_getter(control): # def fn(): # return control['value'] # return fn # set_ctl = make_setter(control) # get_ctl = make_getter(control) # #now we add controls # if control['type']=='bool': # c = ui.Switch(ctl_name,getter=get_ctl,setter=set_ctl) # elif control['type']=='int': # c = ui.Slider(ctl_name,getter=get_ctl,min=control['min'],max=control['max'], # step=control['step'], setter=set_ctl) # elif control['type']=="menu": # if control['menu'] is None: # selection = range(control['min'],control['max']+1,control['step']) # labels = selection # else: # selection = [value for name,value in control['menu'].iteritems()] # labels = [name for name,value in control['menu'].iteritems()] # c = ui.Selector(ctl_name,getter=get_ctl,selection=selection,labels = labels,setter=set_ctl) # else: # pass # if control['disabled']: # c.read_only = True # if ctl_name == 'Exposure, Auto Priority': # # the controll should always be off. we set it to 0 on init (see above) # c.read_only = True # if c is not None: # self.menu.append(c) # self.menu.append(ui.Button("refresh",gui_update_from_device)) # self.menu.append(ui.Button("load defaults",gui_load_defaults)) self.menu.collapsed = True self.sidebar = sidebar #add below geneal settings self.sidebar.insert(1,self.menu)
def ui_elements(self): ui_elements = [] if self.uvc_capture is None: ui_elements.append(ui.Info_Text("Local USB: camera disconnected!")) return ui_elements ui_elements.append(ui.Info_Text(f"Camera: {self.name} @ Local USB")) # lets define some helper functions: def gui_load_defaults(): for c in self.uvc_capture.controls: try: c.value = c.def_val except Exception: pass def gui_update_from_device(): for c in self.uvc_capture.controls: c.refresh() def set_frame_size(new_size): self.frame_size = new_size def set_frame_rate(new_rate): self.frame_rate = new_rate self.update_menu() sensor_control = ui.Growing_Menu(label="Sensor Settings") sensor_control.append( ui.Info_Text("Do not change these during calibration or recording!") ) sensor_control.collapsed = False image_processing = ui.Growing_Menu(label="Image Post Processing") image_processing.collapsed = True sensor_control.append( ui.Selector( "frame_size", self, setter=set_frame_size, selection=self.uvc_capture.frame_sizes, label="Resolution", ) ) def frame_rate_getter(): return ( self.uvc_capture.frame_rates, [str(fr) for fr in self.uvc_capture.frame_rates], ) # TODO: potential race condition through selection_getter. Should ensure that # current selection will always be present in the list returned by the # selection_getter. Highly unlikely though as this needs to happen between # having clicked the Selector and the next redraw. # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b sensor_control.append( ui.Selector( "frame_rate", self, selection_getter=frame_rate_getter, setter=set_frame_rate, label="Frame rate", ) ) if ( "Pupil Cam2" in self.uvc_capture.name or "Pupil Cam3" in self.uvc_capture.name ): special_settings = {200: 28, 180: 31} def set_exposure_mode(exposure_mode): self.exposure_mode = exposure_mode if self.exposure_mode == "auto": self.preferred_exposure_time = Exposure_Time( max_ET=special_settings.get(self.frame_rate, 32), frame_rate=self.frame_rate, mode=self.exposure_mode, ) else: self.preferred_exposure_time = None logger.info( "Exposure mode for camera {0} is now set to {1} mode".format( self.uvc_capture.name, exposure_mode ) ) self.update_menu() sensor_control.append( ui.Selector( "exposure_mode", self, setter=set_exposure_mode, selection=["manual", "auto"], labels=["manual mode", "auto mode"], label="Exposure Mode", ) ) sensor_control.append( ui.Slider( "exposure_time", self, label="Absolute Exposure Time", min=1, max=special_settings.get(self.frame_rate, 32), step=1, ) ) if self.exposure_mode == "auto": sensor_control[-1].read_only = True if "Pupil Cam" in self.uvc_capture.name: blacklist = [ "Auto Focus", "Absolute Focus", "Absolute Iris ", "Scanning Mode ", "Zoom absolute control", "Pan control", "Tilt control", "Roll absolute control", "Privacy Shutter control", ] else: blacklist = [] if ( "Pupil Cam2" in self.uvc_capture.name or "Pupil Cam3" in self.uvc_capture.name ): blacklist += [ "Auto Exposure Mode", "Auto Exposure Priority", "Absolute Exposure Time", ] for control in self.uvc_capture.controls: c = None ctl_name = control.display_name if ctl_name in blacklist: continue # now we add controls if control.d_type == bool: c = ui.Switch( "value", control, label=ctl_name, on_val=control.max_val, off_val=control.min_val, ) elif control.d_type == int: c = ui.Slider( "value", control, label=ctl_name, min=control.min_val, max=control.max_val, step=control.step, ) elif type(control.d_type) == dict: selection = [value for name, value in control.d_type.items()] labels = [name for name, value in control.d_type.items()] c = ui.Selector( "value", control, label=ctl_name, selection=selection, labels=labels ) else: pass # if control['disabled']: # c.read_only = True # if ctl_name == 'Exposure, Auto Priority': # # the controll should always be off. we set it to 0 on init (see above) # c.read_only = True if c is not None: if control.unit == "processing_unit": image_processing.append(c) else: sensor_control.append(c) ui_elements.append(sensor_control) if image_processing.elements: ui_elements.append(image_processing) ui_elements.append(ui.Button("refresh", gui_update_from_device)) if "Pupil Cam2" in self.uvc_capture.name: def set_check_stripes(enable_stripe_checks): self.enable_stripe_checks = enable_stripe_checks if self.enable_stripe_checks: self.stripe_detector = Check_Frame_Stripes() logger.info( "Check Stripes for camera {} is now on".format( self.uvc_capture.name ) ) else: self.stripe_detector = None logger.info( "Check Stripes for camera {} is now off".format( self.uvc_capture.name ) ) ui_elements.append( ui.Switch( "enable_stripe_checks", self, setter=set_check_stripes, label="Check Stripes", ) ) return ui_elements
def _not_valid_video_elements(self): video_path = self.overlay().config.video_path video_name = os.path.basename(video_path) return ( ui.Info_Text("{} was not recorded or cannot be found.".format(video_name)), )
def __init__( self, g_pool, window_size=window_size_default, window_position=None, gui_scale=1.0, ui_config={}, ): super().__init__(g_pool) self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128 glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if g_pool.hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window main_window = glfw.create_window(*window_size, "Pupil Service", None, None) window_position_manager = gl_utils.WindowPositionManager() window_position = window_position_manager.new_window_position( window=main_window, default_position=window_position_default, previous_position=window_position, ) glfw.set_window_pos(main_window, *window_position) glfw.make_context_current(main_window) cygl.utils.init() g_pool.main_window = main_window g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(0, 0), size=(0, 0), header_pos="headline") g_pool.gui.append(g_pool.menubar) # Callback functions def on_resize(window, w, h): # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) self.window_size = w, h self.content_scale = gl_utils.get_content_scale(window) g_pool.gui.scale = self.content_scale g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() # Needed, to update the window buffer while resizing self.update_ui() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = gl_utils.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def set_window_size(): # Get default window size f_width, f_height = window_size_default # Get current display scale factor content_scale = gl_utils.get_content_scale(main_window) framebuffer_scale = gl_utils.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.set_window_size(main_window, int(f_width), int(f_height)) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.set_window_should_close(main_window, True) self.notify_all({"subject": "clear_settings_process.should_start"}) self.notify_all({ "subject": "service_process.should_start", "delay": 2.0 }) g_pool.menubar.append(ui.Button("Reset window size", set_window_size)) pupil_remote_addr = "{}:{}".format( socket.gethostbyname(socket.gethostname()), g_pool.preferred_remote_port) g_pool.menubar.append( ui.Text_Input( "pupil_remote_addr", getter=lambda: pupil_remote_addr, setter=lambda x: None, label="Pupil Remote address", )) g_pool.menubar.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: self.start_stop_eye(0, alive), getter=lambda: g_pool.eye_procs_alive[0].value, )) g_pool.menubar.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: self.start_stop_eye(1, alive), getter=lambda: g_pool.eye_procs_alive[1].value, )) g_pool.menubar.append( ui.Info_Text("Service Version: {}".format(g_pool.version))) g_pool.menubar.append( ui.Button("Restart with default settings", reset_restart)) # Register callbacks main_window glfw.set_framebuffer_size_callback(main_window, on_resize) glfw.set_key_callback(main_window, on_window_key) glfw.set_char_callback(main_window, on_window_char) glfw.set_mouse_button_callback(main_window, on_window_mouse_button) glfw.set_cursor_pos_callback(main_window, on_pos) glfw.set_scroll_callback(main_window, on_scroll) g_pool.gui.configuration = ui_config gl_utils.basic_gl_setup() on_resize(g_pool.main_window, *glfw.get_framebuffer_size(main_window))
def init_ui(self): super().init_ui() self.menu.label = "Manual Calibration" self.menu.append( ui.Info_Text("Calibrate gaze parameters using a handheld marker."))
def world(g_pool, cap_src, cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from eye process[es] Can run various plug-ins. """ #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {})] # Callback functions def on_resize(window, w, h): if not g_pool.iconified: g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': cap_size, 'frame_rate': 24} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.pupil_confidence_threshold = session_settings.get( 'pupil_confidence_threshold', .6) g_pool.active_calibration_plugin = None def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() #window and gl setup glfwInit() width, height = session_settings.get('window_size', (frame.width, frame.height)) main_window = glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize(main_window, frame.width, frame.height))) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=[ p.__name__.replace('_', ' ') for p in user_launchable_plugins ], setter=open_plugin, getter=lambda: "Select to load")) general_settings.append( ui.Slider('pupil_confidence_threshold', g_pool, step=.01, min=0., max=1., label='Minimum pupil confidence')) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append( ui.Hot_Key("quit", setter=on_close, getter=lambda: True, label="X", hotkey=GLFW_KEY_ESCAPE)) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.notifications = [] g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetWindowIconifyCallback(main_window, on_iconify) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) # refresh speed settings glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 130) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Stopping.") break except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() recent_pupil_positions.append(p) pupil_graph.add(p['confidence']) events['pupil_positions'] = recent_pupil_positions # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) if g_pool.iconified: pass else: g_pool.image_tex.update_from_frame(frame) make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() if not g_pool.iconified: graph.push_view() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() graph.pop_view() g_pool.gui.update() glfwSwapBuffers(main_window) glfwPollEvents() glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings[ 'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfwDestroyWindow(main_window) glfwTerminate() cap.close() logger.debug("Process done")
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None,): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy pupil_detectors = { Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D, Detector_Dummy.__name__: Detector_Dummy, } # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.process = "eye{}".format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x *= hdpi_factor y *= hdpi_factor g_pool.gui.update_mouse(x, y) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, g_pool.capture.frame_size) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] g_pool.capture_manager.on_drop(paths) g_pool.capture.on_drop(paths) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id)) ) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.iconified = False/ g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get("flip", False) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } capture_manager_settings = session_settings.get( "capture_manager_settings", ("UVC_Manager", {}) ) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__: c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name]( g_pool, **manager_settings ) if eye_id == 0: cap_src = ["Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"] else: cap_src = ["Pupil Cam2 ID1", "Pupil Cam1 ID1"] # Initialize capture default_settings = ( "UVC_Source", {"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120}, ) capture_source_settings = overwrite_cap_settings or session_settings.get( "capture_settings", default_settings ) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__: c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get("roi") if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get("pupil_detector_settings", None) last_pupil_detector = pupil_detectors[ session_settings.get("last_pupil_detector", Detector_2D.__name__) ] g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_ui() def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = g_pool.capture.frame_size width *= 2 height *= 2 width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2], label="Interface Size", ) ) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width *= 2 f_height *= 2 f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) def uroi_on_mouse_button(button, action, mods): if g_pool.display_mode == "roi": if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: x, y = glfw.glfwGetCursorPos(main_window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) x *= hdpi_factor y *= hdpi_factor pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size ) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size ): # if the roi interacts we dont want # the gui to interact as well return general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", ) ) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode] ) general_settings.append(g_pool.display_mode_info) detector_selector = ui.Selector( "pupil_detector", getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[Detector_Dummy, Detector_2D, Detector_3D], labels=["disabled", "C++ 2d detector", "C++ 3d detector"], label="Detection method", ) general_settings.append(detector_selector) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) toggle_general_settings(False) g_pool.pupil_detector.init_ui() g_pool.capture.init_ui() g_pool.capture_manager.init_ui() g_pool.writer = None def replace_source(source_class_name, source_settings): g_pool.capture.deinit_ui() g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name]( g_pool, **source_settings ) g_pool.capture.init_ui() if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "set_detection_mapping_mode": if notification["mode"] == "3d": if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True elif notification["mode"] == "2d": if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False else: if not isinstance(g_pool.pupil_detector, Detector_Dummy): set_detector(Detector_Dummy) detector_selector.read_only = True elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: record_path = notification["rec_path"] raw_mode = notification["compression"] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join( record_path, "eye{}.mp4".format(eye_id) ) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, g_pool.capture.frame_rate ) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = AV_Writer( video_path, g_pool.capture.frame_rate ) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify( { "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, } ) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif ( subject.startswith("start_eye_capture") and notification["target"] == g_pool.process ): replace_source(notification["name"], notification["args"]) elif notification["subject"].startswith("pupil_detector.set_property"): target_process = notification.get("target", g_pool.process) should_apply = target_process == g_pool.process if should_apply: try: property_name = notification["name"] property_value = notification["value"] if "2d" in notification["subject"]: g_pool.pupil_detector.set_2d_detector_property( property_name, property_value ) elif "3d" in notification["subject"]: if not isinstance(g_pool.pupil_detector, Detector_3D): raise ValueError( "3d properties are only available" " if 3d detector is active" ) g_pool.pupil_detector.set_3d_detector_property( property_name, property_value ) else: raise KeyError( "Notification subject does not " "specifiy detector type." ) logger.debug( "`{}` property set to {}".format( property_name, property_value ) ) except KeyError: logger.error("Malformed notification received") logger.debug(traceback.format_exc()) except (ValueError, TypeError): logger.error("Invalid property or value") logger.debug(traceback.format_exc()) elif notification["subject"].startswith( "pupil_detector.broadcast_properties" ): target_process = notification.get("target", g_pool.process) should_respond = target_process == g_pool.process if should_respond: props = g_pool.pupil_detector.get_detector_properties() properties_broadcast = { "subject": "pupil_detector.properties.{}".format(eye_id), **props, # add properties to broadcast } ipc_socket.notify(properties_broadcast) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get("frame") g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != ( f_height, f_width, ): g_pool.pupil_detector.on_resolution_change( (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size, ) g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'.format( type(frame), frame_publish_format ) ) else: frame_publish_format_recent_warning = False pupil_socket.send( { "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], } ) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect( frame, g_pool.u_r, g_pool.display_mode == "algorithm" ) if result is not None: result["id"] = eye_id result["topic"] = "pupil.{}".format(eye_id) pupil_socket.send(result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == "algorithm": g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ("camera_image", "roi"): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass glViewport(0, 0, *camera_render_size) make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame and result: if result["method"] == "3d c++": eye_ball = result["projected_sphere"] try: pts = cv2.ellipse2Poly( ( int(eye_ball["center"][0]), int(eye_ball["center"][1]), ), ( int(eye_ball["axes"][0] / 2), int(eye_ball["axes"][1] / 2), ), int(eye_ball["angle"]), 0, 360, 8, ) except ValueError as e: pass else: draw_polyline( pts, 2, RGBA(0.0, 0.9, 0.1, result["model_confidence"]), ) if result["confidence"] > 0: if "ellipse" in result: pts = cv2.ellipse2Poly( ( int(result["ellipse"]["center"][0]), int(result["ellipse"]["center"][1]), ), ( int(result["ellipse"]["axes"][0] / 2), int(result["ellipse"]["axes"][1] / 2), ), int(result["ellipse"]["angle"]), 0, 360, 15, ) confidence = result["confidence"] * 0.7 draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence)) draw_points( [result["ellipse"]["center"]], size=20, color=RGBA(1.0, 0.0, 0.0, confidence), sharpness=1.0, ) glViewport(0, 0, *camera_render_size) make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == "roi": g_pool.u_r.draw_points(g_pool.gui.scale) glViewport(0, 0, *window_size) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI unused_elements = g_pool.gui.update() for butt in unused_elements.buttons: uroi_on_mouse_button(*butt) make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip) g_pool.pupil_detector.visualize() # detector decides if we visualize or not # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["roi"] = g_pool.u_r.get() session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["capture_settings"] = (/ g_pool.capture.class_name, g_pool.capture.get_init_dict(), ) session_settings["capture_manager_settings"] = ( g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict(), ) session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_settings[ "last_pupil_detector" ] = g_pool.pupil_detector.__class__.__name__ session_settings[ "pupil_detector_settings" ] = g_pool.pupil_detector.get_settings() session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() g_pool.capture.deinit_ui() g_pool.capture_manager.deinit_ui() g_pool.pupil_detector.deinit_ui() g_pool.pupil_detector.cleanup() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def init_ui(self): super().init_ui() self.menu.label = "Gaze Data From Recording" self.menu.append(ui.Info_Text('Currently, gaze positions are loaded from the recording.'))
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] # logger.setLevel(logging.DEBUG) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { 'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id, 'delay': 0.2 } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) try: # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( '1.9'), 'pyglui out of date, please upgrade to newest version' from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin from object_detector_app import Object_Detection from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze_2 import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes, Base_Manager, Base_Source from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer #from vis_circle import Vis_Circle #custom plugin # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Object_Detection ] system_plugins = [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs ] + manager_classes + source_classes plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins user_plugins += [ p for p in runtime_plugins if not isinstance(p, (Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin)) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { 'preferred_names': [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e" ], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Plugin_Manager', {}), ('System_Graphs', {})] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] for p in g_pool.plugins: p.on_drop(paths) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add( g_pool.plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'stop_plugin': for p in g_pool.plugins: if p.class_name == n['name']: p.alive = False g_pool.plugins.clean() elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) # window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (1280 + icon_bar_width, 720)) main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = camera_render_size[0] + \ int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \ glfw.glfwGetFramebufferSize(main_window)[1] logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'world_process.should_start', 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.6, .8, 1., 1.2, 1.4], label='Interface size')) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings.append(ui.Button('Reset window size', set_window_size)) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) general_settings.append( ui.Info_Text('Capture Version: {}'.format(g_pool.version))) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(False) # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) #a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events['dt'] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # send new events to ipc: del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'depth_frame' in events: del events['depth_frame'] if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) unused_elements = g_pool.gui.update() for button, action, mods in unused_elements.buttons: pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings[ 'detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error('Process Capture crashed with trace:\n{}'.format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) sleep(1.0)
def eye( timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None, hide_ui=False, debug=False, pub_socket_hwm=None, ): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing ``start_eye_plugin``: Start plugins in eye process Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url, pub_socket_hwm) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) if is_alive_flag.value: # indicates eye process that this is a duplicated startup logger.warning("Aborting redundant eye process startup") return with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger): # general imports import traceback import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture import gl_utils from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible, glViewport # monitoring import psutil # Plug-ins from plugin import Plugin_List # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError from ndsi import H264Writer from video_capture import source_classes, manager_classes from roi import Roi from background_helper import IPC_Logging_Task_Proxy from pupil_detector_plugins import available_detector_plugins, EVENT_KEY IPC_Logging_Task_Proxy.push_url = ipc_push_url def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) # NOTE: Interrupt is handled in world/service/player which are responsible for # shutting down the eye process properly signal.signal(signal.SIGINT, interrupt_handler) # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (600, 300 * eye_id + 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (600, 90 + 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) icon_bar_width = 50 window_size = None content_scale = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() # make some constants avaiable g_pool.debug = debug g_pool.user_dir = user_dir g_pool.version = version g_pool.app = "capture" g_pool.eye_id = eye_id g_pool.process = f"eye{eye_id}" g_pool.timebase = timebase g_pool.camera_render_size = None g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic default_2d, default_3d, available_detectors = available_detector_plugins( ) plugins = manager_classes + source_classes + available_detectors + [ Roi ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ f"Pupil Cam3 ID{eye_id}", f"Pupil Cam2 ID{eye_id}", f"Pupil Cam1 ID{eye_id}", ] if eye_id == 0: preferred_names += ["HD-6000"] default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": preferred_names, "frame_size": (192, 192), "frame_rate": 120, } default_plugins = [ # TODO: extend with plugins (default_capture_name, default_capture_settings), ("UVC_Manager", {}), # Detectors needs to be loaded first to set `g_pool.pupil_detector` (default_2d.__name__, {}), (default_3d.__name__, {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), ("File_Manager", {}), ("PupilDetectorManager", {}), ("Roi", {}), ] def consume_events_and_render_buffer(): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() glViewport(0, 0, *g_pool.camera_render_size) for p in g_pool.plugins: p.gl_display() glViewport(0, 0, *window_size) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = glfw.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # update screen glfw.glfwSwapBuffers(main_window) # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal content_scale # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) content_scale = glfw.get_content_scale(window) framebuffer_scale = glfw.get_framebuffer_scale(window) g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = content_scale g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) # Minimum window size required, otherwise parts of the UI can cause openGL # issues with permanent effects. Depends on the content scale, which can # potentially be dynamically modified, so we re-adjust the size limits every # time here. min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) glfw.glfwSetWindowSizeLimits(window, min_size, min_size, glfw.GLFW_DONT_CARE, glfw.GLFW_DONT_CARE) # Needed, to update the window buffer while resizing consume_events_and_render_buffer() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = glfw.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for plugin in g_pool.plugins: if plugin.on_drop(paths): break # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() camera_is_physically_flipped = eye_id == 0 g_pool.iconified = False g_pool.capture = None g_pool.flip = session_settings.get("flip", camera_is_physically_flipped) g_pool.display_mode = session_settings.get("display_mode", "camera_image") g_pool.display_mode_info_text = { "camera_image": "Raw eye camera image. This uses the least amount of CPU power", "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.", } def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # Initialize glfw glfw.glfwInit() glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) if hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window title = "Pupil Capture - eye {}".format(eye_id) # Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions. # As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3 # cameras, we adjust the default eye window size to a 1:1 content aspect ratio. # The size of 500 was chosen s.t. the menu still fits. default_window_size = 500 + icon_bar_width, 500 width, height = session_settings.get("window_size", default_window_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray( np.ones((1, 1), dtype=np.uint8) + 125) # setup GUI g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") def set_window_size(): # Get current capture frame size f_width, f_height = g_pool.capture.frame_size # Eye camera resolutions are too small to be used as default window sizes. # We use double their size instead. frame_scale_factor = 2 f_width *= frame_scale_factor f_height *= frame_scale_factor # Get current display scale factor content_scale = glfw.get_content_scale(main_window) framebuffer_scale = glfw.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Increas the width to account for the added scaled icon bar width f_width += icon_bar_width * display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.glfwSetWindowSize(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Switch("flip", g_pool, label="Flip image display")) general_settings.append( ui.Selector( "display_mode", g_pool, setter=set_display_mode_info, selection=["camera_image", "roi", "algorithm"], labels=["Camera Image", "ROI", "Algorithm"], label="Mode", )) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) plugins_to_load = session_settings.get("loaded_plugins", default_plugins) if overwrite_cap_settings: # Ensure that overwrite_cap_settings takes preference over source plugins # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) g_pool.plugins = Plugin_List(g_pool, plugins_to_load) if not g_pool.capture: # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add(g_pool.plugin_by_name[default_capture_name], default_capture_settings) toggle_general_settings(True) g_pool.writer = None # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # If previously selected plugin was not loaded this time, we will have an # expanded menubar without any menu selected. We need to ensure the menubar is # collapsed in this case. if all(submenu.collapsed for submenu in g_pool.menubar.elements): g_pool.menubar.collapsed = True # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 50) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = "CPU %0.1f" fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 50) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = "jpeg" frame_publish_format_recent_warning = False # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning("Process started.") frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification["subject"] if subject.startswith("eye_process.should_stop"): if notification["eye_id"] == eye_id: break elif subject == "recording.started": if notification["record_eye"] and g_pool.capture.online: record_path = notification["rec_path"] raw_mode = notification["compression"] start_time_synced = notification["start_time_synced"] logger.info( "Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer( video_path, start_time_synced) elif hasattr(g_pool.capture._recent_frame, "h264_buffer"): g_pool.writer = H264Writer( video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate, ) else: g_pool.writer = MPEG_Writer( video_path, start_time_synced) elif subject == "recording.stopped": if g_pool.writer: logger.info("Done recording.") try: g_pool.writer.release() except RuntimeError: logger.error("No eye video recorded") g_pool.writer = None elif subject.startswith("meta.should_doc"): ipc_socket.notify({ "subject": "meta.doc", "actor": "eye{}".format(eye_id), "doc": eye.__doc__, }) elif subject.startswith("frame_publishing.started"): should_publish_frames = True frame_publish_format = notification.get("format", "jpeg") elif subject.startswith("frame_publishing.stopped"): should_publish_frames = False frame_publish_format = "jpeg" elif (subject.startswith("start_eye_plugin") and notification["target"] == g_pool.process): try: g_pool.plugins.add( g_pool.plugin_by_name[notification["name"]], notification.get("args", {}), ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") for plugin in g_pool.plugins: plugin.on_notify(notification) event = {} for plugin in g_pool.plugins: plugin.recent_events(event) frame = event.get("frame") if frame: if should_publish_frames: try: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray assert data is not None except (AttributeError, AssertionError, NameError): if not frame_publish_format_recent_warning: frame_publish_format_recent_warning = True logger.warning( '{}s are not compatible with format "{}"'. format(type(frame), frame_publish_format)) else: frame_publish_format_recent_warning = False pupil_socket.send({ "topic": "frame.eye.{}".format(eye_id), "width": frame.width, "height": frame.height, "index": frame.index, "timestamp": frame.timestamp, "format": frame_publish_format, "__raw_data__": [data], }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1.0 / dt) except ZeroDivisionError: pass if g_pool.writer: try: g_pool.writer.write_video_frame(frame) except NonMonotonicTimestampError as e: logger.error( "Recorder received non-monotonic timestamp!" " Stopping the recording!") logger.debug(str(e)) ipc_socket.notify({"subject": "recording.should_stop"}) ipc_socket.notify({ "subject": "recording.should_stop", "remote_notify": "all" }) for result in event.get(EVENT_KEY, ()): pupil_socket.send(result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): consume_events_and_render_buffer() glfw.glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer.release() g_pool.writer = None session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() # save session persistent settings session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) if not hide_ui: glfw.glfwRestoreWindow( main_window) # need to do this for windows os session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: f_width, f_height = session_window_size if platform.system() in ("Windows", "Linux"): # Store unscaled window size as the operating system will scale the # windows appropriately during launch on Windows and Linux. f_width, f_height = ( f_width / content_scale, f_height / content_scale, ) session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() for plugin in g_pool.plugins: plugin.alive = False g_pool.plugins.clean() glfw.glfwDestroyWindow(main_window) g_pool.gui.terminate() glfw.glfwTerminate() logger.info("Process shutting down.")
def session(rec_dir): plugin_dir = os.path.join(user_dir, 'plugins') if not os.path.isdir(plugin_dir): os.mkdir(plugin_dir) runtime_plugins = import_runtime_plugins(plugin_dir) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__) analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__) other_plugins = sorted([Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__) user_plugins = sorted(runtime_plugins, key=lambda x: x.__name__) user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) global rec_dir rec_dir = new_rec_dir glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) update_recording_to_recent(rec_dir) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) app_version = get_version(version_file) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) < get_version(version_file): logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.pupil_data = pupil_data g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index()-2) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.delayed_notifications[notification['subject']] = notification def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark) export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(export_range.start, export_range.stop)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.notifications.append(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) selector_label = "Select to load" vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins] analysis_labels = [" " + p.__name__.replace('_', ' ') for p in analysis_plugins] other_labels = [" " + p.__name__.replace('_', ' ') for p in other_plugins] user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins] plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label] + other_plugins + [selector_label] + user_plugins) labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"] + other_labels + ["User added"] + user_labels) g_pool.main_menu.append(ui.Selector('Open plugin:', selection=plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.notifications = [] g_pool.delayed_notifications = {} g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfwSetFramebufferSizeCallback(main_window, on_resize) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfwGetFramebufferSize(main_window)) while not glfwWindowShouldClose(main_window): # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # publish delayed notifiactions when their time has come. for n in list(g_pool.delayed_notifications.values()): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() g_pool.gui.update() # present frames at appropriate speed cap.wait(frame) glfwSwapBuffers(main_window) glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfwDestroyWindow(main_window)