def update_gui_markers(self): def set_min_marker_perimeter(val): self.min_marker_perimeter = val self.notify_all({ 'subject': 'min_marker_perimeter_changed', 'delay': 1 }) def set_invert_image(val): self.invert_image = val self.invalidate_marker_cache() self.invalidate_surface_caches() self.menu.elements[:] = [] self.menu.append( ui.Switch('invert_image', self, setter=set_invert_image, label='Use inverted markers')) self.menu.append( ui.Slider('min_marker_perimeter', self, min=20, max=500, step=1, setter=set_min_marker_perimeter)) self.menu.append( ui.Info_Text( 'The offline surface tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.' )) self.menu.append( ui.Info_Text( "Press the export button or type 'e' to start the export.")) self.menu.append( ui.Selector('mode', self, label='Mode', selection=[ "Show Markers and Surfaces", "Show marker IDs", "Show Heatmaps", "Show Metrics" ])) self.menu.append( ui.Info_Text( 'To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.' )) self.menu.append( ui.Button("(Re)-calculate gaze distributions", self.recalculate)) self.menu.append(ui.Button("Add surface", lambda: self.add_surface())) for s in self.surfaces: idx = self.surfaces.index(s) s_menu = ui.Growing_Menu("Surface {}".format(idx)) s_menu.collapsed = True s_menu.append(ui.Text_Input('name', s)) s_menu.append(ui.Text_Input('x', s.real_world_size, label='X size')) s_menu.append(ui.Text_Input('y', s.real_world_size, label='Y size')) s_menu.append(ui.Button('Open Debug Window', s.open_close_window)) #closure to encapsulate idx def make_remove_s(i): return lambda: self.remove_surface(i) remove_s = make_remove_s(idx) s_menu.append(ui.Button('remove', remove_s)) self.menu.append(s_menu)
def init_gui(self): self.menu = ui.Scrolling_Menu('Fixation Detector') self.g_pool.gui.append(self.menu) def set_h_fov(new_fov): self.h_fov = new_fov self.pix_per_degree = float( self.g_pool.capture.frame_size[0]) / new_fov self.notify_all({ 'subject': 'fixations_should_recalculate', 'delay': 1. }) def set_v_fov(new_fov): self.v_fov = new_fov self.pix_per_degree = float( self.g_pool.capture.frame_size[1]) / new_fov self.notify_all({ 'subject': 'fixations_should_recalculate', 'delay': 1. }) def set_duration(new_value): self.min_duration = new_value self.notify_all({ 'subject': 'fixations_should_recalculate', 'delay': 1. }) def set_dispersion(new_value): self.max_dispersion = new_value self.notify_all({ 'subject': 'fixations_should_recalculate', 'delay': 1. }) def jump_next_fixation(_): ts = self.g_pool.capture.get_timestamp() for f in self.fixations: if f['timestamp'] > ts: self.g_pool.capture.seek_to_frame(f['mid_frame_index']) self.g_pool.new_seek = True return logger.error('could not seek to next fixation.') self.menu.append(ui.Button('Close', self.close)) self.menu.append( ui.Info_Text( 'This plugin detects fixations based on a dispersion threshold in terms of degrees of visual angle. It also uses a min duration threshold.' )) self.menu.append( ui.Info_Text( "Press the export button or type 'e' to start the export.")) self.menu.append( ui.Slider('min_duration', self, min=0.0, step=0.05, max=1.0, label='duration threshold', setter=set_duration)) self.menu.append( ui.Slider('max_dispersion', self, min=0.0, step=0.05, max=3.0, label='dispersion threshold', setter=set_dispersion)) self.menu.append( ui.Switch('show_fixations', self, label='Show fixations')) self.menu.append( ui.Slider('h_fov', self, min=5, step=1, max=180, label='horizontal FOV of scene camera', setter=set_h_fov)) self.menu.append( ui.Slider('v_fov', self, min=5, step=1, max=180, label='vertical FOV of scene camera', setter=set_v_fov)) self.add_button = ui.Thumb('jump_next_fixation', setter=jump_next_fixation, getter=lambda: False, label='f', hotkey='f') self.g_pool.quickbar.append(self.add_button)
def __init__( self, g_pool, window_size=window_size_default, window_position=window_position_default, gui_scale=1.0, ui_config={}, ): super().__init__(g_pool) self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128 glfw.glfwInit() glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) if g_pool.hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window main_window = glfw.glfwCreateWindow(*window_size, "Pupil Service") glfw.glfwSetWindowPos(main_window, *window_position) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window g_pool.gui = ui.UI() g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(0, 0), size=(0, 0), header_pos="headline") g_pool.gui.append(g_pool.menubar) # Callback functions def on_resize(window, w, h): # Always clear buffers on resize to make sure that there are no overlapping # artifacts from previous frames. gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT) gl_utils.glClearColor(0, 0, 0, 1) self.window_size = w, h self.content_scale = glfw.get_content_scale(window) g_pool.gui.scale = self.content_scale g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() # Needed, to update the window buffer while resizing self.update_ui() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = glfw.window_coordinate_to_framebuffer_coordinate( window, x, y, cached_scale=None) g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def set_window_size(): # Get default window size f_width, f_height = window_size_default # Get current display scale factor content_scale = glfw.get_content_scale(main_window) framebuffer_scale = glfw.get_framebuffer_scale(main_window) display_scale_factor = content_scale / framebuffer_scale # Scale the capture frame size by display scale factor f_width *= display_scale_factor f_height *= display_scale_factor # Set the newly calculated size (scaled capture frame size + scaled icon bar width) glfw.glfwSetWindowSize(main_window, int(f_width), int(f_height)) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) self.notify_all({"subject": "clear_settings_process.should_start"}) self.notify_all({ "subject": "service_process.should_start", "delay": 2.0 }) g_pool.menubar.append(ui.Button("Reset window size", set_window_size)) pupil_remote_addr = "{}:{}".format( socket.gethostbyname(socket.gethostname()), g_pool.preferred_remote_port) g_pool.menubar.append( ui.Text_Input( "pupil_remote_addr", getter=lambda: pupil_remote_addr, setter=lambda x: None, label="Pupil Remote address", )) g_pool.menubar.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: self.start_stop_eye(0, alive), getter=lambda: g_pool.eye_procs_alive[0].value, )) g_pool.menubar.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: self.start_stop_eye(1, alive), getter=lambda: g_pool.eye_procs_alive[1].value, )) g_pool.menubar.append( ui.Info_Text("Service Version: {}".format(g_pool.version))) g_pool.menubar.append( ui.Button("Restart with default settings", reset_restart)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) g_pool.gui.configuration = ui_config gl_utils.basic_gl_setup() on_resize(g_pool.main_window, *glfw.glfwGetFramebufferSize(main_window))
def init_gui(self, sidebar): sorted_controls = [c for c in self.controls.itervalues()] sorted_controls.sort(key=lambda c: c.order) self.menu = ui.Growing_Menu(label='Camera Settings') cameras = Camera_List() camera_names = [c.name for c in cameras] camera_ids = [c.src_id for c in cameras] self.menu.append( ui.Selector('src_id', self, selection=camera_ids, labels=camera_names, label='Capture Device', setter=self.re_init_cam_by_src_id)) hardware_ts_switch = ui.Switch('hardware_timestamps', None, getter=lambda: False, label='use hardware timestamps') hardware_ts_switch.read_only = True self.menu.append(hardware_ts_switch) for control in sorted_controls: name = control.pretty_name c = None if control.type == "bool": c = ui.Switch('value', control, setter=control.set_val, label=name) elif control.type == 'int': c = ui.Slider('value', control, min=control.min, max=control.max, step=control.step, setter=control.set_val, label=name) elif control.type == "menu": if control.menu is None: selection = range(control.min, control.max + 1, control.step) labels = selection else: #this is currenlty not implemented selection = [c.val for c in control.menu] labels = [c.name for c in control.menu] c = ui.Selector('value', control, selection=selection, labels=labels, label=name, setter=control.set_val) else: pass # print control.type # if control.flags == "inactive": # c.read_only = True if c is not None: self.menu.append(c) self.menu.append(ui.Button("refresh", self.controls.update_from_device)) self.menu.append( ui.Button("load defaults", self.controls.load_defaults)) self.sidebar = sidebar #add below geneal settings self.sidebar.insert(1, self.menu)
def customize_menu(self): self.menu.label = "Eye Video Exporter" self.menu.append(ui.Switch("render_pupil", self, label="Render detected pupil")) super().customize_menu()
def add_controls_to_menu(self, menu, controls): from pyglui import ui # closure factory def make_value_change_fn(ctrl_id): def initiate_value_change(val): logger.debug('%s: %s >> %s' % (self.sensor, ctrl_id, val)) self.sensor.set_control_value(ctrl_id, val) return initiate_value_change for ctrl_id, ctrl_dict in controls: try: dtype = ctrl_dict['dtype'] ctrl_ui = None if dtype == "string": ctrl_ui = ui.Text_Input( 'value', ctrl_dict, label=ctrl_dict['caption'], setter=make_value_change_fn(ctrl_id)) elif dtype == "integer" or dtype == "float": convert_fn = int if dtype == "integer" else float ctrl_ui = ui.Slider( 'value', ctrl_dict, label=ctrl_dict['caption'], min=convert_fn(ctrl_dict.get('min', 0)), max=convert_fn(ctrl_dict.get('max', 100)), step=convert_fn(ctrl_dict.get('res', 0.)), setter=make_value_change_fn(ctrl_id)) elif dtype == "bool": ctrl_ui = ui.Switch('value', ctrl_dict, label=ctrl_dict['caption'], on_val=ctrl_dict.get('max', True), off_val=ctrl_dict.get('min', False), setter=make_value_change_fn(ctrl_id)) elif dtype == "strmapping" or dtype == "intmapping": desc_list = ctrl_dict['map'] labels = [desc['caption'] for desc in desc_list] selection = [desc['value'] for desc in desc_list] ctrl_ui = ui.Selector('value', ctrl_dict, label=ctrl_dict['caption'], labels=labels, selection=selection, setter=make_value_change_fn(ctrl_id)) if ctrl_ui: ctrl_ui.read_only = ctrl_dict.get('readonly', False) self.control_id_ui_mapping[ctrl_id] = ctrl_ui menu.append(ctrl_ui) else: logger.error('Did not generate UI for %s' % ctrl_id) except: logger.error('Exception for control:\n%s' % pprint.pformat(ctrl_dict)) import traceback as tb tb.print_exc() if len(menu) == 0: menu.append(ui.Info_Text("No %s settings found" % menu.label)) return menu
def init_ui(self): self.add_menu() self.menu_icon.order = 0.01 self.menu.label = "System Graphs" self.menu.append(ui.Switch("show_cpu", self, label="Display CPU usage")) self.menu.append( ui.Switch("show_fps", self, label="Display frames per second")) self.menu.append( ui.Switch("show_conf0", self, label="Display confidence for eye 0")) self.menu.append( ui.Switch("show_conf1", self, label="Display confidence for eye 1")) self.menu.append( ui.Switch("show_dia0", self, label="Display pupil diameter for eye 0")) self.menu.append( ui.Switch("show_dia1", self, label="Display pupil diameter for eye 1")) # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) self.cpu_graph = graph.Bar_Graph() self.cpu_graph.pos = (20, 50) self.cpu_graph.update_fn = ps.cpu_percent self.cpu_graph.update_rate = 5 self.cpu_graph.label = "CPU %0.1f" self.fps_graph = graph.Bar_Graph() self.fps_graph.pos = (140, 50) self.fps_graph.update_rate = 5 self.fps_graph.label = "%0.0f FPS" self.conf0_graph = graph.Bar_Graph(max_val=1.0) self.conf0_graph.pos = (260, 50) self.conf0_graph.update_rate = 5 self.conf0_graph.label = "id0 conf: %0.2f" self.conf1_graph = graph.Bar_Graph(max_val=1.0) self.conf1_graph.pos = (380, 50) self.conf1_graph.update_rate = 5 self.conf1_graph.label = "id1 conf: %0.2f" self.dia0_graph = graph.Bar_Graph(min_val=self.dia_min, max_val=self.dia_max) self.dia0_graph.pos = (260, 100) self.dia0_graph.update_rate = 5 self.dia0_graph.label = "id0 dia: %0.2f" self.dia1_graph = graph.Bar_Graph(min_val=self.dia_min, max_val=self.dia_max) self.dia1_graph.pos = (380, 100) self.dia1_graph.update_rate = 5 self.dia1_graph.label = "id1 dia: %0.2f" self.conf_grad = ( RGBA(1.0, 0.0, 0.0, self.conf0_graph.color[3]), self.conf0_graph.color, ) def set_dia_min(val): self.dia0_graph.min_val = val self.dia1_graph.min_val = val def set_dia_max(val): self.dia0_graph.max_val = val self.dia1_graph.max_val = val self.menu.append( ui.Slider( "min_val", self.dia0_graph, label="Minimum pupil diameter", setter=set_dia_min, min=0.0, max=15.0, step=0.1, )) self.menu.append( ui.Slider( "max_val", self.dia0_graph, label="Maximum pupil diameter", setter=set_dia_max, min=1.0, max=15.0, step=0.1, )) self.on_window_resize(self.g_pool.main_window)
def init_ui(self): self.add_menu() self.menu.label = "Recorder" self.menu_icon.order = 0.29 self.menu.append( ui.Info_Text( 'Pupil recordings are saved like this: "path_to_recordings/recording_session_name/nnn" where "nnn" is an increasing number to avoid overwrites. You can use "/" in your session name to create subdirectories.' ) ) self.menu.append( ui.Info_Text( 'Recordings are saved to "~/pupil_recordings". You can change the path here but note that invalid input will be ignored.' ) ) self.menu.append( ui.Text_Input( "rec_root_dir", self, setter=self.set_rec_root_dir, label="Path to recordings", ) ) self.menu.append( ui.Text_Input( "session_name", self, setter=self.set_session_name, label="Recording session name", ) ) self.menu.append( ui.Switch( "show_info_menu", self, on_val=True, off_val=False, label="Request additional user info", ) ) self.menu.append( ui.Selector( "raw_jpeg", self, selection=[True, False], labels=["bigger file, less CPU", "smaller file, more CPU"], label="Compression", ) ) self.menu.append( ui.Info_Text( "Recording the raw eye video is optional. We use it for debugging." ) ) self.menu.append( ui.Switch( "record_eye", self, on_val=True, off_val=False, label="Record eye" ) ) self.button = ui.Thumb( "running", self, setter=self.toggle, label="R", hotkey="r" ) self.button.on_color[:] = (1, 0.0, 0.0, 0.8) self.g_pool.quickbar.insert(2, self.button) self.low_disk_space_thumb = ui.Thumb( "low_disk_warn", label="!", getter=lambda: True, setter=lambda x: None ) self.low_disk_space_thumb.on_color[:] = (1, 0.0, 0.0, 0.8) self.low_disk_space_thumb.status_text = "Low disk space"
def eye(g_pool, cap_src, cap_size, rx_from_world, eye_id=0): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ # modify the root logger for this process logger = logging.getLogger() # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye%s.log' % eye_id), mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level + 10) # create formatter and add it to the handlers formatter = logging.Formatter( 'Eye' + str(eye_id) + ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'EYE' + str(eye_id) + ' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # create logger for the context of this function logger = logging.getLogger(__name__) # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize( window)[0] w, h = w * hdpi_factor, h * hdpi_factor g_pool.gui.update_window(w, h) graph.adjust_size(w, h) adjust_gl_view(w, h) # for p in g_pool.plugins: # p.on_window_resize(window,w,h) glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == GLFW_RELEASE and u_r.active_edit_pt: u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == GLFW_PRESS: pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize( pos, (frame.width, frame.height)) # Position in img pixels if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40, u_r.handle_size + 40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float( glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if u_r.active_edit_pt: pos = normalize((x, y), glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] pos = denormalize(pos, (frame.width, frame.height)) u_r.move_vertex(u_r.active_pt_idx, pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id)) # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 30, timebase=g_pool.timebase) # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return g_pool.capture = cap g_pool.flip = session_settings.get('flip', False) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.window_size = session_settings.get('window_size', 1.) g_pool.display_mode = session_settings.get('display_mode', 'camera_image') g_pool.display_mode_info_text = { 'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below." } # g_pool.draw_pupil = session_settings.get('draw_pupil',True) u_r = UIRoi(frame.img.shape) u_r.set(session_settings.get('roi', u_r.get())) writer = None pupil_detector = Canny_Detector(g_pool) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def get_scale(): return g_pool.gui.scale def set_display_mode_info(val): # set info text here and append to the general settings menu # 'camera_image','roi','algorithm','cpu_save' g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] width, height = session_settings.get('window_size', (frame.width, frame.height)) window_pos = session_settings.get('window_position', (0, 0)) # not yet using this one. # Initialize glfw glfwInit() if g_pool.binocular: title = "Binocular eye %s" % eye_id else: title = 'Eye' main_window = glfwCreateWindow(width, height, title, None, None) glfwMakeContextCurrent(main_window) cygl_init() # Register callbacks main_window glfwSetWindowSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img.shape) update_named_texture(g_pool.image_tex, frame.img) # refresh speed settings glfwSwapInterval(0) glfwSetWindowPos(main_window, 800, 300 * eye_id + window_position_default[1]) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') g_pool.sidebar.configuration = session_settings.get( 'side_bar_config', {'collapsed': True}) general_settings = ui.Growing_Menu('General') general_settings.configuration = session_settings.get( 'general_menu_config', {}) general_settings.append( ui.Slider('scale', setter=set_scale, getter=get_scale, step=.05, min=1., max=2.5, label='Interface Size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfwSetWindowSize(main_window, frame.width, frame.height))) general_settings.append( ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image', 'roi', 'algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode")) general_settings.append( ui.Switch('flip', g_pool, label='Flip image display')) g_pool.display_mode_info = ui.Info_Text( g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append( ui.Hot_Key("quit", setter=on_close, getter=lambda: True, label="X", hotkey=GLFW_KEY_ESCAPE)) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.menu.configuration = session_settings.get( 'capture_menu_config', {'collapsed': True}) # let detector add its GUI pupil_detector.init_gui(g_pool.sidebar) #set the last saved window size on_resize(main_window, *glfwGetWindowSize(main_window)) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.get_cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if rx_from_world.poll(): command = rx_from_world.recv() if command is not None: record_path = command logger.info("Will save eye video to: %s" % record_path) video_path = os.path.join(record_path, "eye%s.mkv" % eye_id) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy" % eye_id) writer = cv2.VideoWriter( video_path, cv2.cv.CV_FOURCC(*'DIVX'), float(cap.frame_rate), (frame.img.shape[1], frame.img.shape[0])) timestamps = [] else: logger.info("Done recording.") writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write(frame.img) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect( frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': update_named_texture(g_pool.image_tex, frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): update_named_texture(g_pool.image_tex, frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) draw_named_texture(g_pool.image_tex) # switch to work in pixel space make_coord_system_pixel_based((frame.height, frame.width, 3), g_pool.flip) if result['confidence'] > 0: if result.has_key('axes'): pts = cv2.ellipse2Poly( (int(result['center'][0]), int(result['center'][1])), (int(result['axes'][0] / 2), int(result['axes'][1] / 2)), int(result['angle']), 0, 360, 15) cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5)) cygl_draw_points([result['center']], size=20, color=cygl_rgba(1., 0., 0., .5), sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': u_r.draw(g_pool.gui.scale) #update screen glfwSwapBuffers(main_window) glfwPollEvents() # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['side_bar_config'] = g_pool.sidebar.configuration session_settings['capture_menu_config'] = g_pool.capture.menu.configuration session_settings['general_menu_config'] = general_settings.configuration session_settings['window_size'] = glfwGetWindowSize(main_window) session_settings['window_position'] = glfwGetWindowPos(main_window) session_settings.close() pupil_detector.cleanup() cap.close() glfwDestroyWindow(main_window) glfwTerminate() #flushing queue in case world process did not exit gracefully while not g_pool.pupil_queue.empty(): g_pool.pupil_queue.get() g_pool.pupil_queue.close() logger.debug("Process done")
def __init__(self, g_pool, window_size=window_size_default, window_position=window_position_default, gui_scale=1., ui_config={}): super().__init__(g_pool) self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128 glfw.glfwInit() main_window = glfw.glfwCreateWindow(*window_size, "Pupil Service") glfw.glfwSetWindowPos(main_window, *window_position) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window g_pool.gui = ui.UI() g_pool.gui_user_scale = gui_scale g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(0, 0), size=(0, 0), header_pos='headline') g_pool.gui.append(g_pool.menubar) # Callback functions def on_resize(window, w, h): self.window_size = w, h self.hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * self.hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * self.hdpi_factor, y * self.hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *self.window_size) def set_window_size(): glfw.glfwSetWindowSize(main_window, *window_size_default) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) self.notify_all({'subject': 'clear_settings_process.should_start'}) self.notify_all({ 'subject': 'service_process.should_start', 'delay': 2. }) g_pool.menubar.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.6, .8, 1., 1.2, 1.4], label='Interface size')) g_pool.menubar.append(ui.Button('Reset window size', set_window_size)) pupil_remote_addr = '{}:50020'.format( socket.gethostbyname(socket.gethostname())) g_pool.menubar.append( ui.Text_Input('pupil_remote_addr', getter=lambda: pupil_remote_addr, setter=lambda x: None, label='Pupil Remote address')) g_pool.menubar.append( ui.Selector('detection_mapping_mode', g_pool, label='Detection & mapping mode', setter=self.set_detection_mapping_mode, selection=['disabled', '2d', '3d'])) g_pool.menubar.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: self.start_stop_eye(0, alive), getter=lambda: g_pool.eyes_are_alive[0].value)) g_pool.menubar.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: self.start_stop_eye(1, alive), getter=lambda: g_pool.eyes_are_alive[1].value)) g_pool.menubar.append( ui.Info_Text('Service Version: {}'.format(g_pool.version))) g_pool.menubar.append( ui.Button('Restart with default settings', reset_restart)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) g_pool.gui.configuration = ui_config gl_utils.basic_gl_setup() on_resize(g_pool.main_window, *glfw.glfwGetFramebufferSize(main_window))
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {"subject": "set_detection_mapping_mode", "mode": new_mode} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( "1.24"), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_routines import ( calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin, ) from fixation_detector import Fixation_Detector from eye_movement import Eye_Movement_Detector_Real_Time from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins")) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Eye_Movement_Detector_Real_Time, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, Online_Head_Pose_Tracker, ] system_plugins = ([ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes) plugins = (system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ ("UVC_Source", default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), ("Screen_Marker_Calibration", {}), ("Recorder", {}), ("Pupil_Remote", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for plugin in g_pool.plugins: if plugin.on_drop(paths): break tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world")) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8) g_pool.detection_mapping_mode = session_settings.get( "detection_mapping_mode", "3d") g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode) def handle_notifications(noti): subject = noti["subject"] if subject == "set_detection_mapping_mode": if noti["mode"] == "2d": if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add( g_pool.plugin_by_name["Dummy_Gaze_Mapper"]) g_pool.detection_mapping_mode = noti["mode"] elif subject == "start_plugin": g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})) elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_detection_mapping_mode", "mode": g_pool.detection_mapping_mode, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) elif subject == "world_process.adapt_window_size": set_window_size() width, height = session_settings.get("window_size", (1280 + icon_bar_width, 720)) # window and gl setup glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "world_process.should_start", "delay": 2.0 }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.6, 0.8, 1.0, 1.2, 1.4], label="Interface size", )) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) on_resize(main_window, f_width, f_height) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector("audio_mode", audio, selection=audio.audio_modes)) general_settings.append( ui.Selector( "detection_mapping_mode", g_pool, label="detection & mapping mode", setter=set_detection_mapping_mode, selection=["disabled", "2d", "3d"], )) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, )) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, )) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version))) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins glfw.glfwPollEvents() if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.glfwSwapBuffers(main_window) glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings[ "detection_mapping_mode"] = g_pool.detection_mapping_mode session_settings["audio_mode"] = audio.audio_mode session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def ui_elements(self): ui_elements = [] if self.uvc_capture is None: ui_elements.append(ui.Info_Text("Local USB: camera disconnected!")) return ui_elements ui_elements.append(ui.Info_Text(f"Camera: {self.name} @ Local USB")) # lets define some helper functions: def gui_load_defaults(): for c in self.uvc_capture.controls: try: c.value = c.def_val except Exception: pass def gui_update_from_device(): for c in self.uvc_capture.controls: c.refresh() def set_frame_size(new_size): self.frame_size = new_size def set_frame_rate(new_rate): self.frame_rate = new_rate self.update_menu() sensor_control = ui.Growing_Menu(label="Sensor Settings") sensor_control.append( ui.Info_Text( "Do not change these during calibration or recording!")) sensor_control.collapsed = False image_processing = ui.Growing_Menu(label="Image Post Processing") image_processing.collapsed = True sensor_control.append( ui.Selector( "frame_size", self, setter=set_frame_size, selection=self.uvc_capture.frame_sizes, label="Resolution", )) def frame_rate_getter(): return ( self.uvc_capture.frame_rates, [str(fr) for fr in self.uvc_capture.frame_rates], ) # TODO: potential race condition through selection_getter. Should ensure that # current selection will always be present in the list returned by the # selection_getter. Highly unlikely though as this needs to happen between # having clicked the Selector and the next redraw. # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b sensor_control.append( ui.Selector( "frame_rate", self, selection_getter=frame_rate_getter, setter=set_frame_rate, label="Frame rate", )) if ("Pupil Cam2" in self.uvc_capture.name or "Pupil Cam3" in self.uvc_capture.name): special_settings = {200: 28, 180: 31} def set_exposure_mode(exposure_mode): self.exposure_mode = exposure_mode if self.exposure_mode == "auto": self.preferred_exposure_time = Exposure_Time( max_ET=special_settings.get(self.frame_rate, 32), frame_rate=self.frame_rate, mode=self.exposure_mode, ) else: self.preferred_exposure_time = None logger.info( "Exposure mode for camera {0} is now set to {1} mode". format(self.uvc_capture.name, exposure_mode)) self.update_menu() sensor_control.append( ui.Selector( "exposure_mode", self, setter=set_exposure_mode, selection=["manual", "auto"], labels=["manual mode", "auto mode"], label="Exposure Mode", )) sensor_control.append( ui.Slider( "exposure_time", self, label="Absolute Exposure Time", min=1, max=special_settings.get(self.frame_rate, 32), step=1, )) if self.exposure_mode == "auto": sensor_control[-1].read_only = True if "Pupil Cam" in self.uvc_capture.name: blacklist = [ "Auto Focus", "Absolute Focus", "Absolute Iris ", "Scanning Mode ", "Zoom absolute control", "Pan control", "Tilt control", "Roll absolute control", "Privacy Shutter control", ] else: blacklist = [] if ("Pupil Cam2" in self.uvc_capture.name or "Pupil Cam3" in self.uvc_capture.name): blacklist += [ "Auto Exposure Mode", "Auto Exposure Priority", "Absolute Exposure Time", ] for control in self.uvc_capture.controls: c = None ctl_name = control.display_name if ctl_name in blacklist: continue # now we add controls if control.d_type == bool: c = ui.Switch( "value", control, label=ctl_name, on_val=control.max_val, off_val=control.min_val, ) elif control.d_type == int: c = ui.Slider( "value", control, label=ctl_name, min=control.min_val, max=control.max_val, step=control.step, ) elif type(control.d_type) == dict: selection = [value for name, value in control.d_type.items()] labels = [name for name, value in control.d_type.items()] c = ui.Selector("value", control, label=ctl_name, selection=selection, labels=labels) else: pass # if control['disabled']: # c.read_only = True # if ctl_name == 'Exposure, Auto Priority': # # the controll should always be off. we set it to 0 on init (see above) # c.read_only = True if c is not None: if control.unit == "processing_unit": image_processing.append(c) else: sensor_control.append(c) ui_elements.append(sensor_control) if image_processing.elements: ui_elements.append(image_processing) ui_elements.append(ui.Button("refresh", gui_update_from_device)) if "Pupil Cam2" in self.uvc_capture.name: def set_check_stripes(enable_stripe_checks): self.enable_stripe_checks = enable_stripe_checks if self.enable_stripe_checks: self.stripe_detector = Check_Frame_Stripes() logger.info("Check Stripes for camera {} is now on".format( self.uvc_capture.name)) else: self.stripe_detector = None logger.info( "Check Stripes for camera {} is now off".format( self.uvc_capture.name)) ui_elements.append( ui.Switch( "enable_stripe_checks", self, setter=set_check_stripes, label="Check Stripes", )) return ui_elements
def init_ui(self): self.add_menu() self.menu.label = 'Accuracy Visualizer' mapping_error_help = '''The mapping error (orange line) is the angular distance between mapped pupil positions (red) and their corresponding reference points (blue). '''.replace("\n", " ").replace(" ", '') calib_area_help = '''The calibration area (green) is defined as the convex hull of the reference points that were used for calibration. 2D mapping looses accuracy outside of this area. It is recommended to calibrate a big portion of the subject's field of view. '''.replace("\n", " ").replace(" ", '') self.menu.append(ui.Info_Text(calib_area_help)) self.menu.append( ui.Switch('vis_mapping_error', self, label='Visualize mapping error')) self.menu.append(ui.Info_Text(mapping_error_help)) self.menu.append( ui.Switch('vis_calibration_area', self, label='Visualize calibration area')) general_help = '''Measure gaze mapping accuracy and precision using samples that were collected during calibration. The outlier threshold discards samples with high angular errors.'''.replace( "\n", " ").replace(" ", '') self.menu.append(ui.Info_Text(general_help)) # self.menu.append(ui.Info_Text('')) self.menu.append( ui.Text_Input('outlier_threshold', self, label='Outlier Threshold [degrees]')) accuracy_help = '''Accuracy is calculated as the average angular offset (distance) (in degrees of visual angle) between fixation locations and the corresponding locations of the fixation targets.'''.replace( "\n", " ").replace(" ", '') precision_help = '''Precision is calculated as the Root Mean Square (RMS) of the angular distance (in degrees of visual angle) between successive samples during a fixation.'''.replace( "\n", " ").replace(" ", '') def ignore(_): pass self.menu.append(ui.Info_Text(accuracy_help)) self.menu.append( ui.Text_Input('accuracy', self, 'Angular Accuracy', setter=ignore, getter=lambda: self.accuracy if self.accuracy is not None else 'Not available')) self.menu.append(ui.Info_Text(precision_help)) self.menu.append( ui.Text_Input('precision', self, 'Angular Precision', setter=ignore, getter=lambda: self.precision if self.precision is not None else 'Not available'))
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, cap_src): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. #general imports from time import time, sleep import numpy as np import logging import zmq import zmq_tools #zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('pupil', )) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) #log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) #display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush, is_window_visible #check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version assert pyglui_version >= '1.0' #monitoring import psutil # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from version_utils import VersionFormat import audio from uvc import get_time_monotonic #trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins from fixation_detector import Fixation_Detector_3D from recorder import Recorder from show_calibration import Show_Calibration from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher logger.info('Application Version: %s' % version) logger.info('System Info: %s' % get_system_info()) #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) #g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic #manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_launchable_plugins = [ Pupil_Groups, Frame_Publisher, Show_Calibration, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector_3D ] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder] plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Fixation_Detector_3D', {})] # Callback functions def on_resize(window, w, h): if is_window_visible(window): g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() graph.adjust_size(w, h) adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if session_settings.get("version", VersionFormat('0.0')) < g_pool.version: logger.info( "Session setting are from older version of this app. I will not use those." ) session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size': (1280, 720), 'frame_rate': 30} previous_settings = session_settings.get('capture_settings', None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings g_pool.iconified = False g_pool.capture = cap g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '2d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.%s' % eye_id, 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name: logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify: ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) #window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', cap.frame_size) main_window = glfw.glfwCreateWindow(width, height, "World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale', 1) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append( ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=1., max=2.5, label='Interface size')) general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, frame.width, frame.height))) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) selector_label = "Select to load" labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append( ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append( ui.Info_Text('Capture Version: %s' % g_pool.version)) g_pool.sidebar.append(general_settings) g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.quickbar) g_pool.capture.init_gui(g_pool.sidebar) #plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert( 0, ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection=calibration_plugins, labels=[p.__name__.replace('_', ' ') for p in calibration_plugins], setter=open_plugin, label='Method')) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() # refresh speed settings glfw.glfwSwapInterval(0) #trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) #now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) #set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = cap.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # Get an image from the grabber try: frame = g_pool.capture.get_frame() except CameraCaptureError: logger.error("Capture from camera failed. Starting Fake Capture.") settings = g_pool.capture.settings g_pool.capture.close() g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase) g_pool.capture.init_gui(g_pool.sidebar) g_pool.capture.settings = settings ipc_pub.notify({'subject': 'recording.should_stop'}) continue except EndofVideoFileError: logger.warning("Video file is done. Stopping") break #update performace graphs t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1. / dt) except ZeroDivisionError: pass cpu_graph.update() #a dictionary that allows plugins to post and read events events = {} #report time between now and the last loop interation events['dt'] = get_dt() recent_pupil_data = [] recent_gaze_data = [] new_notifications = [] while pupil_sub.new_data: t, p = pupil_sub.recv() pupil_graphs[p['id']].add(p['confidence']) recent_pupil_data.append(p) new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p) for g in new_gaze_data: gaze_pub.send('gaze', g) recent_gaze_data += new_gaze_data while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) events['pupil_positions'] = recent_pupil_data events['gaze_positions'] = recent_gaze_data # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) #check if a plugin need to be destroyed g_pool.plugins.clean() #send new events to ipc: del events['pupil_positions'] #already on the wire del events['gaze_positions'] #send earlier in this loop del events['dt'] #no need to send this for topic, data in events.iteritems(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) # render camera image glfw.glfwMakeContextCurrent(main_window) if is_window_visible(main_window): g_pool.image_tex.update_from_frame(frame) glFlush() make_coord_system_norm_based() g_pool.image_tex.draw() make_coord_system_pixel_based((frame.height, frame.width, 3)) # render visual feedback from loaded plugins if is_window_visible(main_window): for p in g_pool.plugins: p.gl_display() graph.push_view() fps_graph.draw() cpu_graph.draw() pupil0_graph.draw() pupil1_graph.draw() graph.pop_view() g_pool.gui.update() glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) #need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui.scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture.close() #shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) #shut down launcher n = {'subject': 'launcher_process.should_stop'} ipc_pub.notify(n) zmq_ctx.destroy()
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # display import glfw from pyglui import ui, graph, cygl, __version__ as pyglui_version assert pyglui_version >= '1.7' from pyglui.cygl.utils import Named_Texture import gl_utils # monitoring import psutil # helpers/utils from version_utils import VersionFormat from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes, Base_Manager from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer from diameter_history import Diameter_History # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (0, 0) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) calibration_plugins += [p for p in runtime_plugins if issubclass(p, Calibration_Plugin)] runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Calibration_Plugin)] manager_classes += [p for p in runtime_plugins if issubclass(p, Base_Manager)] runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Base_Manager)] user_launchable_plugins = [Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector, Diameter_History, Blink_Detection, Remote_Recorder, Accuracy_Visualizer] + runtime_plugins system_plugins = [Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay] plugin_by_index = (system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes + source_classes) name_by_index = [p.__name__ for p in plugin_by_index] plugin_by_name = dict(zip(name_by_index, plugin_by_index)) default_capture_settings = { 'preferred_names': ["Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e"], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {})] # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconified): g_pool.iconified = iconified def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0]) x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] for p in g_pool.plugins: p.on_drop(paths) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info("Session setting are from a different version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture_manager = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def launch_eye_process(eye_id, delay=0): n = {'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay} ipc_pub.notify(n) def stop_eye_process(eye_id): n = {'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id,'delay':0.2} ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.") g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'stop_plugin': for p in g_pool.plugins: if p.class_name == n['name']: p.alive = False g_pool.plugins.clean() elif subject == 'eye_process.started': n = {'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode} ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) # window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (1280, 720)) main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'reset_restart_process.should_start'}) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Button('Reset to default settings',reset_restart)) general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface size')) general_settings.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window,g_pool.capture.frame_size[0],g_pool.capture.frame_size[1])) ) general_settings.append(ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append(ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d','3d'] )) general_settings.append(ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0,alive), getter=lambda: eyes_are_alive[0].value )) general_settings.append(ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1,alive), getter=lambda: eyes_are_alive[1].value )) selector_label = "Select to load" user_launchable_plugins.sort(key=lambda p: p.__name__) labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins] user_launchable_plugins.insert(0, selector_label) labels.insert(0, selector_label) general_settings.append(ui.Selector('Open plugin', selection=user_launchable_plugins, labels=labels, setter=open_plugin, getter=lambda: selector_label)) general_settings.append(ui.Info_Text('Capture Version: {}'.format(g_pool.version))) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.calibration_menu = ui.Growing_Menu('Calibration') g_pool.calibration_menu.collapsed = True g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.sidebar.append(g_pool.calibration_menu) g_pool.gui.append(g_pool.sidebar) g_pool.gui.append(g_pool.quickbar) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins)) #We add the calibration menu selector, after a calibration has been added: g_pool.calibration_menu.insert(0,ui.Selector( 'active_calibration_plugin', getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins, labels = [p.__name__.replace('_',' ') for p in calibration_plugins], setter= open_plugin,label='Method' )) #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager', setter = open_plugin, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" pupil0_graph = graph.Bar_Graph(max_val=1.0) pupil0_graph.pos = (260, 130) pupil0_graph.update_rate = 5 pupil0_graph.label = "id0 conf: %0.2f" pupil1_graph = graph.Bar_Graph(max_val=1.0) pupil1_graph.pos = (380, 130) pupil1_graph.update_rate = 5 pupil1_graph.label = "id1 conf: %0.2f" pupil_graphs = pupil0_graph, pupil1_graph g_pool.graphs = [cpu_graph, fps_graph, pupil0_graph, pupil1_graph] # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) #a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events['dt'] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # update performace graphs if 'frame' in events: t = events["frame"].timestamp dt, ts = t-ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass for p in events["pupil_positions"]: pupil_graphs[p['id']].add(p['confidence']) cpu_graph.update() # send new events to ipc: del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'depth_frame' in events: del events['depth_frame'] if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert(isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible(main_window): for p in g_pool.plugins: p.gl_display() for g in g_pool.graphs: g.draw() unused_elements = g_pool.gui.update() for button, action, mods in unused_elements.buttons: pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'})
def update_menu(self): del self.menu[:] from pyglui import ui ui_elements = [] # lets define some helper functions: def gui_load_defaults(): for c in self.uvc_capture.controls: try: c.value = c.def_val except: pass def gui_update_from_device(): for c in self.uvc_capture.controls: c.refresh() def set_frame_size(new_size): self.frame_size = new_size if self.uvc_capture is None: ui_elements.append(ui.Info_Text('Capture initialization failed.')) self.menu.extend(ui_elements) return ui_elements.append(ui.Info_Text('{} Controls'.format(self.name))) sensor_control = ui.Growing_Menu(label='Sensor Settings') sensor_control.append(ui.Info_Text("Do not change these during calibration or recording!")) sensor_control.collapsed = False image_processing = ui.Growing_Menu(label='Image Post Processing') image_processing.collapsed = True sensor_control.append(ui.Selector( 'frame_size', self, setter=set_frame_size, selection=self.uvc_capture.frame_sizes, label='Resolution' )) def frame_rate_getter(): return (self.uvc_capture.frame_rates, [str(fr) for fr in self.uvc_capture.frame_rates]) sensor_control.append(ui.Selector('frame_rate', self, selection_getter=frame_rate_getter, label='Frame rate')) for control in self.uvc_capture.controls: c = None ctl_name = control.display_name # now we add controls if control.d_type == bool: c = ui.Switch('value', control, label=ctl_name, on_val=control.max_val, off_val=control.min_val) elif control.d_type == int: c = ui.Slider('value', control, label=ctl_name, min=control.min_val, max=control.max_val, step=control.step) elif type(control.d_type) == dict: selection = [value for name, value in control.d_type.items()] labels = [name for name, value in control.d_type.items()] c = ui.Selector('value', control, label=ctl_name, selection=selection, labels=labels) else: pass # if control['disabled']: # c.read_only = True # if ctl_name == 'Exposure, Auto Priority': # # the controll should always be off. we set it to 0 on init (see above) # c.read_only = True if c is not None: if control.unit == 'processing_unit': image_processing.append(c) else: sensor_control.append(c) ui_elements.append(sensor_control) if image_processing.elements: ui_elements.append(image_processing) ui_elements.append(ui.Button("refresh",gui_update_from_device)) ui_elements.append(ui.Button("load defaults",gui_load_defaults)) self.menu.extend(ui_elements)
def init_ui(self): super().init_ui() self.menu.label = "Offline Calibration" self.glfont = fontstash.Context() self.glfont.add_font("opensans", ui.get_opensans_font_path()) self.glfont.set_color_float((1.0, 1.0, 1.0, 1.0)) self.glfont.set_align_string(v_align="right", h_align="top") def use_as_natural_features(): self.manual_ref_positions.extend(self.circle_marker_positions) self.manual_ref_positions.sort(key=lambda mr: mr["index"]) def jump_next_natural_feature(): self.manual_ref_positions.sort(key=lambda mr: mr["index"]) current = self.g_pool.capture.get_frame_index() for nf in self.manual_ref_positions: if nf["index"] > current: self.notify_all({ "subject": "seek_control.should_seek", "index": nf["index"] }) return logger.error("No further natural feature available") def clear_natural_features(): self.manual_ref_positions = [] self.menu.append( ui.Info_Text( '"Detection" searches for circle markers in the world video.')) # self.menu.append(ui.Button('Redetect', self.start_marker_detection)) slider = ui.Slider("detection_progress", self, label="Detection Progress", setter=lambda _: _) slider.display_format = "%3.0f%%" self.menu.append(slider) toggle_label = ("Cancel circle marker detection" if self.process_pipe else "Start circle marker detection") self.toggle_detection_button = ui.Button(toggle_label, self.toggle_marker_detection) self.menu.append(self.toggle_detection_button) self.menu.append(ui.Separator()) self.menu.append( ui.Button("Use calibration markers as natural features", use_as_natural_features)) self.menu.append( ui.Button("Jump to next natural feature", jump_next_natural_feature)) self.menu.append( ui.Switch("manual_ref_edit_mode", self, label="Natural feature edit mode")) self.menu.append( ui.Button("Clear natural features", clear_natural_features)) self.menu.append( ui.Info_Text( "Calibration only considers pupil data that has an equal or higher confidence than the minimum calibration confidence." )) self.menu.append( ui.Slider( "min_calibration_confidence", self.g_pool, step=0.01, min=0.0, max=1.0, label="Minimum calibration confidence", )) self.menu.append(ui.Button("Add section", self.append_section)) # set to minimum height self.timeline = ui.Timeline("Calibration Sections", self.draw_sections, self.draw_labels, 1) self.g_pool.user_timelines.append(self.timeline) for sec in self.sections: self.append_section_menu(sec) self.on_window_resize( glfw.glfwGetCurrentContext(), *glfw.glfwGetWindowSize(glfw.glfwGetCurrentContext()))
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings=None): """reads eye video and detects the pupil. Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates. Reacts to notifications: ``set_detection_mapping_mode``: Sets detection method ``eye_process.should_stop``: Stops the eye process ``recording.started``: Starts recording eye video ``recording.stopped``: Stops recording eye video ``frame_publishing.started``: Starts frame publishing ``frame_publishing.stopped``: Stops frame publishing Emits notifications: ``eye_process.started``: Eye process started ``eye_process.stopped``: Eye process stopped Emits data: ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>`` ``frame.eye.<eye id>``: Eye frames with id ``<eye id>`` """ # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. import zmq import zmq_tools zmq_ctx = zmq.Context() ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id): # logging setup import logging logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # general imports import numpy as np import cv2 # display import glfw from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based from gl_utils import is_window_visible from ui_roi import UIRoi # monitoring import psutil # helpers/utils from uvc import get_time_monotonic from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, timer from av_writer import JPEG_Writer, AV_Writer from ndsi import H264Writer from video_capture import source_classes from video_capture import manager_classes # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__: Detector_2D, Detector_3D.__name__: Detector_3D} # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600, 300 * eye_id) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (600,31+ 300 * eye_id) else: scroll_factor = 1.0 window_position_default = (600, 300 * eye_id) # g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.process = 'eye{}'.format(eye_id) g_pool.timebase = timebase g_pool.ipc_pub = ipc_socket def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # Callback functions def on_resize(window, w, h): if is_window_visible(window): active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_iconify(window, iconified): g_pool.iconified = iconified def on_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = glfw.glfwGetFramebufferSize( window)[0] / glfw.glfwGetWindowSize(window)[0] g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x, y), glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,g_pool.capture.frame_size ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id))) if VersionFormat(session_settings.get("version", '0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() g_pool.iconified = False g_pool.capture = None g_pool.capture_manager = None g_pool.flip = session_settings.get('flip', False) g_pool.display_mode = session_settings.get( 'display_mode', 'camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} capture_manager_settings = session_settings.get( 'capture_manager_settings', ('UVC_Manager',{})) manager_class_name, manager_settings = capture_manager_settings manager_class_by_name = {c.__name__:c for c in manager_classes} g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings) if eye_id == 0: cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"] else: cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"] # Initialize capture default_settings = ('UVC_Source',{ 'preferred_names' : cap_src, 'frame_size': (640,480), 'frame_rate': 90 }) capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings) source_class_name, source_settings = capture_source_settings source_class_by_name = {c.__name__:c for c in source_classes} g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) assert g_pool.capture g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0])) roi_user_settings = session_settings.get('roi') if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]: g_pool.u_r.set(roi_user_settings) pupil_detector_settings = session_settings.get( 'pupil_detector_settings', None) last_pupil_detector = pupil_detectors[session_settings.get( 'last_pupil_detector', Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector( g_pool, pupil_detector_settings) def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "Pupil Capture - eye {}".format(eye_id) width, height = session_settings.get( 'window_size', g_pool.capture.frame_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get( 'window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # UI callback functions def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125) # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-300, 0), size=(0, 0), header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode', g_pool, setter=set_display_mode_info, selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector', getter=lambda: g_pool.pupil_detector.__class__, setter=set_detector, selection=[ Detector_2D, Detector_3D], labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection') g_pool.capture_source_menu = ui.Growing_Menu('Capture Source') g_pool.capture_source_menu.collapsed = True g_pool.capture.init_gui() g_pool.sidebar.append(general_settings) g_pool.sidebar.append(g_pool.capture_selector_menu) g_pool.sidebar.append(g_pool.capture_source_menu) g_pool.pupil_detector.init_gui(g_pool.sidebar) g_pool.capture_manager.init_gui() g_pool.writer = None def replace_source(source_class_name,source_settings): g_pool.capture.cleanup() g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings) g_pool.capture.init_gui() if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture def replace_manager(manager_class): g_pool.capture_manager.cleanup() g_pool.capture_manager = manager_class(g_pool) g_pool.capture_manager.init_gui() #We add the capture selection menu, after a manager has been added: g_pool.capture_selector_menu.insert(0,ui.Selector( 'capture_manager',g_pool, setter = replace_manager, getter = lambda: g_pool.capture_manager.__class__, selection = manager_classes, labels = [b.gui_name for b in manager_classes], label = 'Manager' )) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetWindowIconifyCallback(main_window, on_iconify) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = g_pool.get_timestamp() cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" g_pool.graphs = [cpu_graph, fps_graph] # set the last saved window size on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) should_publish_frames = False frame_publish_format = 'jpeg' # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) logger.warning('Process started.') frame = None # Event loop while not glfw.glfwWindowShouldClose(main_window): if notify_sub.new_data: t, notification = notify_sub.recv() subject = notification['subject'] if subject == 'eye_process.should_stop': if notification['eye_id'] == eye_id: break elif subject == 'set_detection_mapping_mode': if notification['mode'] == '3d': if not isinstance(g_pool.pupil_detector, Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: if not isinstance(g_pool.pupil_detector, Detector_2D): set_detector(Detector_2D) detector_selector.read_only = False elif subject == 'recording.started': if notification['record_eye'] and g_pool.capture.online: record_path = notification['rec_path'] raw_mode = notification['compression'] logger.info("Will save eye video to: {}".format(record_path)) video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id)) if raw_mode and frame and g_pool.capture.jpeg_support: g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate) elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'): g_pool.writer = H264Writer(video_path, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1], g_pool.capture.frame_rate) else: g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate) elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") g_pool.writer.release() g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({ 'subject': 'meta.doc', 'actor': 'eye{}'.format(eye_id), 'doc': eye.__doc__ }) elif subject.startswith('frame_publishing.started'): should_publish_frames = True frame_publish_format = notification.get('format', 'jpeg') elif subject.startswith('frame_publishing.stopped'): should_publish_frames = False frame_publish_format = 'jpeg' elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process: replace_source(notification['name'],notification['args']) g_pool.capture.on_notify(notification) # Get an image from the grabber event = {} g_pool.capture.recent_events(event) frame = event.get('frame') g_pool.capture_manager.recent_events(event) if frame: f_width, f_height = g_pool.capture.frame_size if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width): g_pool.u_r = UIRoi((f_height, f_width)) if should_publish_frames and frame.jpeg_buffer: if frame_publish_format == "jpeg": data = frame.jpeg_buffer elif frame_publish_format == "yuv": data = frame.yuv_buffer elif frame_publish_format == "bgr": data = frame.bgr elif frame_publish_format == "gray": data = frame.gray pupil_socket.send('frame.eye.%s'%eye_id,{ 'width': frame.width, 'height': frame.width, 'index': frame.index, 'timestamp': frame.timestamp, 'format': frame_publish_format, '__raw_data__': [data] }) t = frame.timestamp dt, ts = t - ts, t try: fps_graph.add(1./dt) except ZeroDivisionError: pass if g_pool.writer: g_pool.writer.write_video_frame(frame) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result pupil_socket.send('pupil.%s'%eye_id,result) cpu_graph.update() # GL drawing if window_should_update(): if is_window_visible(main_window): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if frame: # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image', 'roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() f_width, f_height = g_pool.capture.frame_size make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip) if frame: if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]), int(eye_ball['center'][1])), (int(eye_ball['axes'][0] / 2), int(eye_ball['axes'][1] / 2)), int(eye_ball['angle']), 0, 360, 8) except ValueError as e: pass else: draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence'])) if result['confidence'] > 0: if 'ellipse' in result: pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]), int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0] / 2), int(result['ellipse']['axes'][1] / 2)), int(result['ellipse']['angle']), 0, 360, 15) confidence = result['confidence'] * 0.7 draw_polyline(pts, 1, RGBA(1., 0, 0, confidence)) draw_points([result['ellipse']['center']], size=20, color=RGBA(1., 0., 0., confidence), sharpness=1.) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI g_pool.gui.update() # render the ROI g_pool.u_r.draw(g_pool.gui.scale) if g_pool.display_mode == 'roi': g_pool.u_r.draw_points(g_pool.gui.scale) # update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() # detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if g_pool.writer: logger.info("Done recording eye.") g_pool.writer = None glfw.glfwRestoreWindow(main_window) # need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict() session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict() session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.capture.deinit_gui() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() g_pool.capture_manager.cleanup() g_pool.capture.cleanup() logger.info("Process shutting down.")
def update_menu(self): logger.debug("update_menu") try: del self.menu[:] except AttributeError: return from pyglui import ui if not self.online: self.menu.append(ui.Info_Text("Capture initialization failed.")) return self.menu.append( ui.Switch("record_depth", self, label="Record Depth Stream")) self.menu.append( ui.Switch("preview_depth", self, label="Preview Depth")) if self._available_modes is not None: def frame_size_selection_getter(): if self.device_id: frame_size = sorted(self._available_modes[rs.stream.color], reverse=True) labels = [ "({}, {})".format(t[0], t[1]) for t in frame_size ] return frame_size, labels else: return [self.frame_size_backup ], [str(self.frame_size_backup)] selector = ui.Selector( "frame_size", self, selection_getter=frame_size_selection_getter, label="Color Resolution", ) self.menu.append(selector) def frame_rate_selection_getter(): if self.device_id: avail_fps = [ fps for fps in self._available_modes[rs.stream.color][ self.frame_size] ] return avail_fps, [str(fps) for fps in avail_fps] else: return [self.frame_rate_backup ], [str(self.frame_rate_backup)] selector = ui.Selector( "frame_rate", self, selection_getter=frame_rate_selection_getter, label="Color Frame Rate", ) self.menu.append(selector) def depth_frame_size_selection_getter(): if self.device_id: depth_sizes = sorted( self._available_modes[rs.stream.depth], reverse=True) labels = [ "({}, {})".format(t[0], t[1]) for t in depth_sizes ] return depth_sizes, labels else: return ( [self.depth_frame_size_backup], [str(self.depth_frame_size_backup)], ) selector = ui.Selector( "depth_frame_size", self, selection_getter=depth_frame_size_selection_getter, label="Depth Resolution", ) self.menu.append(selector) def depth_frame_rate_selection_getter(): if self.device_id: avail_fps = [ fps for fps in self._available_modes[rs.stream.depth][ self.depth_frame_size] ] return avail_fps, [str(fps) for fps in avail_fps] else: return ( [self.depth_frame_rate_backup], [str(self.depth_frame_rate_backup)], ) selector = ui.Selector( "depth_frame_rate", self, selection_getter=depth_frame_rate_selection_getter, label="Depth Frame Rate", ) self.menu.append(selector) def reset_options(): logger.debug("reset_options") self.reset_device(self.device_id) sensor_control = ui.Growing_Menu(label="Sensor Settings") sensor_control.append( ui.Button("Reset device options to default", reset_options)) self.menu.append(sensor_control) else: logger.debug("update_menu: self._available_modes is None")