Beispiel #1
0
    def init_ui(self):
        # lets make a menu entry in the sidebar
        self.add_menu()
        self.menu.label = 'My Plugin'

        # and a slider
        self.menu.append(ui.Slider('my_var', self, min=1.0))

        # this is also a good place to open a custom window:
        # let make the size identical to the img size:
        width, height = self.g_pool.capture.frame_size
        # here we use glfw to create a new window: 'share' should point to
        # the main gl context so we can reuse textures from there.
        self.window = glfwCreateWindow(width, height, "Plugin Window", monitor=None, share=glfwGetCurrentContext())

        # now the set up some window callback. Have a look at referece_surface.py to see whats possible
        glfwSetFramebufferSizeCallback(self.window, self.on_resize)

        # lets set up the window gl state
        # if you want to draw or change settings of a window
        # you need to activate it and once done deactivate
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(self.window)
        # set up blending etc
        gl_utils.basic_gl_setup()
        # refresh speed settings
        glfwSwapInterval(0)
        glfwMakeContextCurrent(active_window)

        self.on_resize(self.window, *glfwGetFramebufferSize(self.window))
    def open_window(self,title='new_window'):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx]
                width,height,redBits,blueBits,greenBits,refreshRate = glfwGetVideoMode(monitor)
            else:
                monitor = None
                width,height= 640,360

            self._window = glfwCreateWindow(width, height, title, monitor=monitor, share=glfwGetCurrentContext())
            if not self.fullscreen:
                glfwSetWindowPos(self._window,200,0)

            glfwSetInputMode(self._window,GLFW_CURSOR,GLFW_CURSOR_HIDDEN)

            #Register callbacks
            glfwSetFramebufferSizeCallback(self._window,on_resize)
            glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window,self.on_close)
            glfwSetMouseButtonCallback(self._window,self.on_button)
            on_resize(self._window,*glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)
Beispiel #3
0
 def _gl_state_settings(window):
     active_window = glfw.glfwGetCurrentContext()
     glfw.glfwMakeContextCurrent(window)
     gl_utils.basic_gl_setup()
     gl_utils.make_coord_system_norm_based()
     glfw.glfwSwapInterval(0)
     glfw.glfwMakeContextCurrent(active_window)
    def open_window(self):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx]
                mode = glfwGetVideoMode(monitor)
                # glfwGetFramebufferSize(window)
                hdpi_factor = glfwGetFramebufferSize(glfwGetCurrentContext())[0]/glfwGetWindowSize(glfwGetCurrentContext())[0]
                width,height= mode[0]*hdpi_factor,mode[1]*hdpi_factor
            else:
                monitor = None
                width,height= 640,360

            self._window = glfwCreateWindow(width, height, "Calibration", monitor=monitor, share=glfwGetCurrentContext())
            if not self.fullscreen:
                glfwSetWindowPos(self._window,200,0)

            glfwSetInputMode(self._window,GLFW_CURSOR,GLFW_CURSOR_HIDDEN)
            on_resize(self._window,width,height)

            #Register callbacks
            glfwSetWindowSizeCallback(self._window,on_resize)
            glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window,self.on_close)
            glfwSetMouseButtonCallback(self._window,self.on_button)

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)
Beispiel #5
0
    def open_window(self):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx]
                mode = glfwGetVideoMode(monitor)
                height,width= mode[0],mode[1]
            else:
                monitor = None
                height,width= 640,360

            self._window = glfwCreateWindow(height, width, "Accuracy_Test", monitor=monitor, share=glfwGetCurrentContext())
            if not self.fullscreen:
                glfwSetWindowPos(self._window,200,0)

            on_resize(self._window,height,width)

            #Register callbacks
            glfwSetWindowSizeCallback(self._window,on_resize)
            glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window,self.on_close)

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)
    def open_window(self):
        if not self._window:
            if self.fullscreen.value:
                monitor = self.monitor_handles[self.monitor_idx.value]
                mode = glfwGetVideoMode(monitor)
                height,width= mode[0],mode[1]
            else:
                monitor = None
                height,width= 640,360

            self._window = glfwCreateWindow(height, width, "Calibration", monitor=monitor, share=glfwGetCurrentContext())
            if not self.fullscreen.value:
                glfwSetWindowPos(self._window,200,0)

            on_resize(self._window,height,width)

            #Register callbacks
            glfwSetWindowSizeCallback(self._window,on_resize)
            glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window,self.on_close)

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            glfwMakeContextCurrent(active_window)
            self.window_should_open = False
    def open_window(self):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx]
                mode = glfwGetVideoMode(monitor)
                height,width= mode[0],mode[1]
            else:
                monitor = None
                height,width = 640,480

            self._window = glfwCreateWindow(height, width, "Calibration", monitor=monitor, share=glfwGetCurrentContext())
            if not self.fullscreen:
                glfwSetWindowPos(self._window,200,0)

            on_resize(self._window,height,width)

            #Register callbacks
            glfwSetWindowSizeCallback(self._window,on_resize)
            glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window,self.on_close)
            glfwSetMouseButtonCallback(self._window,self.on_button)


            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            glfwMakeContextCurrent(active_window)

            self.clicks_to_close = 5
Beispiel #8
0
    def open_window(self):
        if not self._window:
            if self.fullscreen.value:
                monitor = glfwGetMonitors()[self.monitor_idx.value]
                mode = glfwGetVideoMode(monitor)
                height,width= mode[0],mode[1]
            else:
                monitor = None
                height,width= 1280,720

            self._window = glfwCreateWindow(height, width, "Reference Surface", monitor=monitor, share=glfwGetCurrentContext())
            if not self.fullscreen.value:
                glfwSetWindowPos(self._window,200,0)

            on_resize(self._window,height,width)

            #Register callbacks
            glfwSetWindowSizeCallback(self._window,on_resize)
            glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window,self.on_close)

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()

            # refresh speed settings
            glfwSwapInterval(0)


            glfwMakeContextCurrent(active_window)

            self.window_should_open = False
Beispiel #9
0
        def toggle_depth_display():
            def on_depth_mouse_button(window, button, action, mods):
                if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS:
                    self.mouse_drag = True
                if (
                    button == glfw.GLFW_MOUSE_BUTTON_LEFT
                    and action == glfw.GLFW_RELEASE
                ):
                    self.mouse_drag = False

            if self.depth_window is None:
                self.pitch = 0
                self.yaw = 0

                win_size = glfw.glfwGetWindowSize(self.g_pool.main_window)
                self.depth_window = glfw.glfwCreateWindow(
                    win_size[0], win_size[1], "3D Point Cloud"
                )
                glfw.glfwSetMouseButtonCallback(
                    self.depth_window, on_depth_mouse_button
                )
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(self.depth_window)
                gl_utils.basic_gl_setup()
                gl_utils.make_coord_system_norm_based()

                # refresh speed settings
                glfw.glfwSwapInterval(0)

                glfw.glfwMakeContextCurrent(active_window)
Beispiel #10
0
    def open_window(self,size):
        if not self._window:
            if 0: #we are not fullscreening
                monitor = glfwGetMonitors()[self.monitor_idx.value]
                mode = glfwGetVideoMode(monitor)
                height,width= mode[0],mode[1]
            else:
                monitor = None
                height,width= size

            active_window = glfwGetCurrentContext()
            self._window = glfwCreateWindow(height, width, "Plugin Window", monitor=monitor, share=active_window)
            if not 0:
                glfwSetWindowPos(self._window,200,0)

            self.on_resize(self._window,height,width)

            #Register callbacks
            glfwSetWindowSizeCallback(self._window,self.on_resize)
            # glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window,self.on_close)

            # gl_state settings
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()

            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)

            self.window_should_open = False
Beispiel #11
0
    def open_window(self):
        if not self._window:

            monitor = None
            # open with same aspect ratio as surface
            surface_aspect_ratio = (
                self.surface.real_world_size["x"] / self.surface.real_world_size["y"]
            )
            win_h = 640
            win_w = int(win_h / surface_aspect_ratio)

            self._window = glfw.glfwCreateWindow(
                win_h,
                win_w,
                "Reference Surface: " + self.surface.name,
                monitor=monitor,
                share=glfw.glfwGetCurrentContext(),
            )

            glfw.glfwSetWindowPos(
                self._window,
                self.window_position_default[0],
                self.window_position_default[1],
            )

            self.trackball = gl_utils.trackball.Trackball()
            self.input = {"down": False, "mouse": (0, 0)}

            # Register callbacks
            glfw.glfwSetFramebufferSizeCallback(self._window, self.on_resize)
            glfw.glfwSetKeyCallback(self._window, self.on_window_key)
            glfw.glfwSetWindowCloseCallback(self._window, self.on_close)
            glfw.glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)
            glfw.glfwSetCursorPosCallback(self._window, self.on_pos)
            glfw.glfwSetScrollCallback(self._window, self.on_scroll)

            self.on_resize(self._window, *glfw.glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfw.glfwGetCurrentContext()
            glfw.glfwMakeContextCurrent(self._window)
            gl_utils.basic_gl_setup()
            gl_utils.make_coord_system_norm_based()

            # refresh speed settings
            glfw.glfwSwapInterval(0)

            glfw.glfwMakeContextCurrent(active_window)
    def open_window(self, title="new_window"):
        if not self._window:
            if self.fullscreen:
                try:
                    monitor = glfwGetMonitors()[self.monitor_idx]
                except:
                    logger.warning(
                        "Monitor at index %s no longer availalbe using default"
                        % self.monitor_idx
                    )
                    self.monitor_idx = 0
                    monitor = glfwGetMonitors()[self.monitor_idx]
                width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(
                    monitor
                )
            else:
                monitor = None
                width, height = 640, 360

            self._window = glfwCreateWindow(
                width, height, title, monitor=monitor, share=glfwGetCurrentContext()
            )
            if not self.fullscreen:
                glfwSetWindowPos(
                    self._window,
                    self.window_position_default[0],
                    self.window_position_default[1],
                )

            glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)

            # Register callbacks
            glfwSetFramebufferSizeCallback(self._window, on_resize)
            glfwSetKeyCallback(self._window, self.on_window_key)
            glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)
            on_resize(self._window, *glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)
    def open_window(self):
        if not self._window:
            if self.fullscreen:
                try:
                    monitor = glfwGetMonitors()[self.monitor_idx]
                except:
                    logger.warning(
                        "Monitor at index %s no longer availalbe using default" % idx
                    )
                    self.monitor_idx = 0
                    monitor = glfwGetMonitors()[self.monitor_idx]
                mode = glfwGetVideoMode(monitor)
                height, width = mode[0], mode[1]
            else:
                monitor = None
                height, width = 640, 480

            self._window = glfwCreateWindow(
                height,
                width,
                "Calibration",
                monitor=monitor,
                share=glfwGetCurrentContext(),
            )
            if not self.fullscreen:
                # move to y = 31 for windows os
                glfwSetWindowPos(self._window, 200, 31)

            # Register callbacks
            glfwSetFramebufferSizeCallback(self._window, on_resize)
            glfwSetKeyCallback(self._window, self.on_window_key)
            glfwSetWindowCloseCallback(self._window, self.on_close)
            glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)

            on_resize(self._window, *glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            glfwMakeContextCurrent(active_window)

            self.clicks_to_close = 5
Beispiel #14
0
    def open_window(self):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx]
                mode = glfwGetVideoMode(monitor)
                height, width = mode[0], mode[1]
            else:
                monitor = None
                height, width = (
                    640,
                    int(640.0 / (self.real_world_size["x"] / self.real_world_size["y"])),
                )  # open with same aspect ratio as surface

            self._window = glfwCreateWindow(
                height, width, "Reference Surface: " + self.name, monitor=monitor, share=glfwGetCurrentContext()
            )
            if not self.fullscreen:
                glfwSetWindowPos(self._window, 200, 0)

            self.trackball = Trackball()
            self.input = {"down": False, "mouse": (0, 0)}

            # Register callbacks
            glfwSetFramebufferSizeCallback(self._window, self.on_resize)
            glfwSetKeyCallback(self._window, self.on_key)
            glfwSetWindowCloseCallback(self._window, self.on_close)
            glfwSetMouseButtonCallback(self._window, self.on_button)
            glfwSetCursorPosCallback(self._window, self.on_pos)
            glfwSetScrollCallback(self._window, self.on_scroll)

            self.on_resize(self._window, *glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            make_coord_system_norm_based()

            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)
    def open_window(self):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx.value]
                mode = glfwGetVideoMode(monitor)
                height, width = mode[0], mode[1]
            else:
                monitor = None
                height, width = (
                    640,
                    int(640.0 / (self.scale_factor[0] / self.scale_factor[1])),
                )  # open with same aspect ratio as surface

            self._window = glfwCreateWindow(
                height, width, "Reference Surface: " + self.name, monitor=monitor, share=glfwGetCurrentContext()
            )
            if not self.fullscreen.value:
                glfwSetWindowPos(self._window, 200, 0)

            self.on_resize(self._window, height, width)

            # Register callbacks
            glfwSetWindowSizeCallback(self._window, self.on_resize)
            glfwSetKeyCallback(self._window, self.on_key)
            glfwSetWindowCloseCallback(self._window, self.on_close)

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            make_coord_system_norm_based()

            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)

            self.window_should_open = False
    def open_window(self, title='new_window'):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx]
                width, height, redBits, blueBits, greenBits, refreshRate = glfwGetVideoMode(
                    monitor)
            else:
                monitor = None
                width, height = 640, 360

            self._window = glfwCreateWindow(width,
                                            height,
                                            title,
                                            monitor=monitor,
                                            share=glfwGetCurrentContext())
            if not self.fullscreen:
                glfwSetWindowPos(self._window, self.window_position_default[0],
                                 self.window_position_default[1])

            glfwSetInputMode(self._window, GLFW_CURSOR, GLFW_CURSOR_HIDDEN)

            # Register callbacks
            glfwSetFramebufferSizeCallback(self._window, on_resize)
            glfwSetKeyCallback(self._window, self.on_window_key)
            glfwSetMouseButtonCallback(self._window,
                                       self.on_window_mouse_button)
            on_resize(self._window, *glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)
Beispiel #17
0
    def open_window(self, size):
        if not self._window:
            if 0:  #we are not fullscreening
                monitor = glfwGetMonitors()[self.monitor_idx.value]
                mode = glfwGetVideoMode(monitor)
                height, width = mode[0], mode[1]
            else:
                monitor = None
                height, width = size

            active_window = glfwGetCurrentContext()
            self._window = glfwCreateWindow(height,
                                            width,
                                            "Plugin Window",
                                            monitor=monitor,
                                            share=active_window)
            if not 0:
                glfwSetWindowPos(self._window, 200, 0)

            self.on_resize(self._window, height, width)

            #Register callbacks
            glfwSetWindowSizeCallback(self._window, self.on_resize)
            # glfwSetKeyCallback(self._window,self.on_key)
            glfwSetWindowCloseCallback(self._window, self.on_close)

            # gl_state settings
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()

            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)

            self.window_should_open = False
Beispiel #18
0
def world(g_pool,cap_src,cap_size):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from g_pool.pupil_queue
    Can run various plug-ins.
    """


    # Callback functions
    def on_resize(window,w, h):
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        norm_size = normalize((w,h),glfwGetWindowSize(window))
        fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
        atb.TwWindowSize(*map(int,fb_size))
        adjust_gl_view(w,h,window)
        glfwMakeContextCurrent(active_window)

    def on_iconify(window,iconfied):
        if not isinstance(cap,FakeCapture):
            g_pool.update_textures.value = not iconfied

    def on_key(window, key, scancode, action, mods):
        if not atb.TwEventKeyboardGLFW(key,action):
            if action == GLFW_PRESS:
                if key == GLFW_KEY_ESCAPE:
                    on_close(window)

    def on_char(window,char):
        if not atb.TwEventCharGLFW(char,1):
            pass

    def on_button(window,button, action, mods):
        if not atb.TwEventMouseButtonGLFW(button,action):
            pos = glfwGetCursorPos(window)
            pos = normalize(pos,glfwGetWindowSize(world_window))
            pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
            for p in g_pool.plugins:
                p.on_click(pos,button,action)

    def on_pos(window,x, y):
        norm_pos = normalize((x,y),glfwGetWindowSize(window))
        fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window))
        if atb.TwMouseMotion(int(fb_x),int(fb_y)):
            pass

    def on_scroll(window,x,y):
        if not atb.TwMouseWheel(int(x)):
            pass

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')



    # load session persistent settings
    session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_world'),protocol=2)
    def load(var_name,default):
        return session_settings.get(var_name,default)
    def save(var_name,var):
        session_settings[var_name] = var


    # load last calibration data
    try:
        pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy'))
        logger.info("Using calibration found in %s" %g_pool.user_dir)
        map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height))
    except:
        logger.info("No calibration found.")
        def map_pupil(vector):
            """ 1 to 1 mapping
            """
            return vector

    # any object we attach to the g_pool object now will only be visible to this process!
    # vars should be declared here to make them visible to the reader.
    g_pool.plugins = []
    g_pool.map_pupil = map_pupil
    g_pool.update_textures = c_bool(1)

    # Initialize capture
    cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)

    if isinstance(cap,FakeCapture):
        g_pool.update_textures.value = False

     # Get an image from the grabber
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    height,width = frame.img.shape[:2]
    g_pool.capture = cap

    # helpers called by the main atb bar
    def update_fps():
        old_time, bar.timestamp = bar.timestamp, time()
        dt = bar.timestamp - old_time
        if dt:
            bar.fps.value += .05 * (1. / dt - bar.fps.value)

    def set_window_size(mode,data):
        height,width = frame.img.shape[:2]
        ratio = (1,.75,.5,.25)[mode]
        w,h = int(width*ratio),int(height*ratio)
        glfwSetWindowSize(world_window,w,h)
        data.value=mode # update the bar.value

    def get_from_data(data):
        """
        helper for atb getter and setter use
        """
        return data.value

    def open_calibration(selection,data):
        # prepare destruction of current ref_detector... and remove it
        for p in g_pool.plugins:
            if isinstance(p,calibration_routines.detector_by_index):
                p.alive = False
        g_pool.plugins = [p for p in g_pool.plugins if p.alive]

        new_ref_detector = calibration_routines.detector_by_index[selection](g_pool,atb_pos=bar.next_atb_pos)
        g_pool.plugins.append(new_ref_detector)
        g_pool.plugins.sort(key=lambda p: p.order)

        # save the value for atb bar
        data.value=selection

    def toggle_record_video():
        for p in g_pool.plugins:
            if isinstance(p,recorder.Recorder):
                p.alive = False
                return
        # set up folder within recordings named by user input in atb
        if not bar.rec_name.value:
            bar.rec_name.value = recorder.get_auto_name()

        new_plugin = recorder.Recorder(g_pool,bar.rec_name.value, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx)
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def toggle_show_calib_result():
        for p in g_pool.plugins:
            if isinstance(p,Show_Calibration):
                p.alive = False
                return

        new_plugin = Show_Calibration(g_pool,frame.img.shape)
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def toggle_server():
        for p in g_pool.plugins:
            if isinstance(p,Pupil_Server):
                p.alive = False
                return

        new_plugin = Pupil_Server(g_pool,(10,300))
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)


    def toggle_ar():
        for p in g_pool.plugins:
            if isinstance(p,Marker_Detector):
                p.alive = False
                return

        new_plugin = Marker_Detector(g_pool,(10,400))
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def reset_timebase():
        #the last frame from worldcam will be t0
        g_pool.timebase.value = cap.get_now()
        logger.info("New timebase set to %s all timestamps will count from here now."%g_pool.timebase.value)

    atb.init()
    # add main controls ATB bar
    bar = atb.Bar(name = "World", label="Controls",
            help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
            text='light', position=(10, 10),refresh=.3, size=(300, 200))
    bar.next_atb_pos = (10,220)
    bar.fps = c_float(0.0)
    bar.timestamp = time()
    bar.calibration_type = c_int(load("calibration_type",0))
    bar.record_eye = c_bool(load("record_eye",0))
    bar.window_size = c_int(load("window_size",0))
    window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
    calibrate_type_enum = atb.enum("Calibration Method",calibration_routines.index_by_name)
    bar.rec_name = create_string_buffer(512)
    bar.version = create_string_buffer(g_pool.version,512)
    bar.rec_name.value = recorder.get_auto_name()
    bar.add_var("fps", bar.fps, step=1., readonly=True, help="Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate.")
    bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size,help="Resize the world window. This has no effect on the actual image.")
    bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.")
    bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.")
    bar.add_var("session name",bar.rec_name, group="Recording", help="Give your recording session a custom name.")
    bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording")
    bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye")
    bar.add_button("start/stop marker tracking",toggle_ar,key="x",help="find markers in scene to map gaze onto referace surfaces")
    bar.add_button("start/stop server",toggle_server,key="s",help="the server broadcasts pupil and gaze positions locally or via network")
    bar.add_button("set timebase to now",reset_timebase,help="this button allows the timestamps to count from now on.",key="t")
    bar.add_var("update screen", g_pool.update_textures,help="if you dont need to see the camera image updated, you can turn this of to reduce CPU load.")
    bar.add_separator("Sep1")
    bar.add_var("version",bar.version, readonly=True)
    bar.add_var("exit", g_pool.quit)

    # add uvc camera controls ATB bar
    cap.create_atb_bar(pos=(320,10))

    # Initialize glfw
    glfwInit()
    world_window = glfwCreateWindow(width, height, "World", None, None)
    glfwMakeContextCurrent(world_window)

    # Register callbacks world_window
    glfwSetWindowSizeCallback(world_window,on_resize)
    glfwSetWindowCloseCallback(world_window,on_close)
    glfwSetWindowIconifyCallback(world_window,on_iconify)
    glfwSetKeyCallback(world_window,on_key)
    glfwSetCharCallback(world_window,on_char)
    glfwSetMouseButtonCallback(world_window,on_button)
    glfwSetCursorPosCallback(world_window,on_pos)
    glfwSetScrollCallback(world_window,on_scroll)

    #set the last saved window size
    set_window_size(bar.window_size.value,bar.window_size)
    on_resize(world_window, *glfwGetWindowSize(world_window))
    glfwSetWindowPos(world_window,0,0)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = create_named_texture(frame.img)
    # refresh speed settings
    glfwSwapInterval(0)


    #load calibration plugin
    open_calibration(bar.calibration_type.value,bar.calibration_type)

    #load gaze_display plugin
    g_pool.plugins.append(Display_Recent_Gaze(g_pool))


    # Event loop
    while not g_pool.quit.value:

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break

        update_fps()

        #a container that allows plugins to post and read events
        events = []

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            if p['norm_pupil'] is None:
                p['norm_gaze'] = None
            else:
                p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil'])
            recent_pupil_positions.append(p)


        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,recent_pupil_positions,events)

        #check if a plugin need to be destroyed
        g_pool.plugins = [p for p in g_pool.plugins if p.alive]


        # render camera image
        glfwMakeContextCurrent(world_window)

        make_coord_system_norm_based()
        if g_pool.update_textures.value:
            draw_named_texture(g_pool.image_tex,frame.img)
        else:
            draw_named_texture(g_pool.image_tex)
        make_coord_system_pixel_based(frame.img.shape)

        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        atb.draw()
        glfwSwapBuffers(world_window)
        glfwPollEvents()


    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
        #reading p.alive actually runs plug-in cleanup
        _ = p.alive

    save('window_size',bar.window_size.value)
    save('calibration_type',bar.calibration_type.value)
    save('record_eye',bar.record_eye.value)
    session_settings.close()

    cap.close()
    atb.terminate()
    glfwDestroyWindow(world_window)
    glfwTerminate()
    logger.debug("Process done")
Beispiel #19
0
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
          user_dir, version):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    # logger.setLevel(logging.DEBUG)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            'subject': 'eye_process.should_start.{}'.format(eye_id),
            'eye_id': eye_id,
            'delay': delay
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            'subject': 'eye_process.should_stop.{}'.format(eye_id),
            'eye_id': eye_id,
            'delay': 0.2
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
        ipc_pub.notify(n)

    try:

        # display
        import glfw
        from version_utils import VersionFormat
        from pyglui import ui, cygl, __version__ as pyglui_version
        assert VersionFormat(pyglui_version) >= VersionFormat(
            '1.9'), 'pyglui out of date, please upgrade to newest version'
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic
        logger.info('Application Version: {}'.format(version))
        logger.info('System Info: {}'.format(get_system_info()))

        import audio

        # trigger pupil detector cpp build:
        import pupil_detectors
        del pupil_detectors

        # Plug-ins
        from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from pupil_remote import Pupil_Remote
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from frame_publisher import Frame_Publisher
        from blink_detection import Blink_Detection
        from video_capture import source_classes, manager_classes, Base_Manager, Base_Source
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from audio_capture import Audio_Capture
        from accuracy_visualizer import Accuracy_Visualizer
        # from saccade_detector import Saccade_Detector
        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay

        # UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == 'Windows':
            scroll_factor = 10.0
            window_position_default = (8, 31)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process they are accesible to all plugins
        g_pool = Global_Container()
        g_pool.app = 'capture'
        g_pool.process = 'world'
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eyes_are_alive = eyes_are_alive

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, 'plugins'))
        user_plugins = [
            Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote,
            Time_Sync, Surface_Tracker, Annotation_Capture, Log_History,
            Fixation_Detector, Blink_Detection, Remote_Recorder,
            Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay
        ]
        system_plugins = [
            Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay,
            Plugin_Manager, System_Graphs
        ] + manager_classes + source_classes
        plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins
        user_plugins += [
            p for p in runtime_plugins
            if not isinstance(p, (Base_Manager, Base_Source,
                                  System_Plugin_Base, Calibration_Plugin,
                                  Gaze_Mapping_Plugin))
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_settings = {
            'preferred_names': [
                "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510",
                "B525", "C525", "C615", "C920", "C930e"
            ],
            'frame_size': (1280, 720),
            'frame_rate':
            30
        }

        default_plugins = [("UVC_Source", default_capture_settings),
                           ('Pupil_Data_Relay', {}), ('UVC_Manager', {}),
                           ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
                           ('Display_Recent_Gaze', {}),
                           ('Screen_Marker_Calibration', {}), ('Recorder', {}),
                           ('Pupil_Remote', {}), ('Plugin_Manager', {}),
                           ('System_Graphs', {})]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode('utf-8') for x in range(count)]
            for p in g_pool.plugins:
                p.on_drop(paths)

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_world'))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.detection_mapping_mode = session_settings.get(
            'detection_mapping_mode', '3d')
        g_pool.active_calibration_plugin = None
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.audio_mode = session_settings.get('audio_mode',
                                                audio.default_audio_mode)

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'set_detection_mapping_mode':
                if n['mode'] == '2d':
                    if ("Vector_Gaze_Mapper"
                            in g_pool.active_gaze_mapping_plugin.class_name):
                        logger.warning(
                            "The gaze mapper is not supported in 2d mode. Please recalibrate."
                        )
                        g_pool.plugins.add(
                            g_pool.plugin_by_name['Dummy_Gaze_Mapper'])
                g_pool.detection_mapping_mode = n['mode']
            elif subject == 'start_plugin':
                g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
                                   args=n.get('args', {}))
            elif subject == 'stop_plugin':
                for p in g_pool.plugins:
                    if p.class_name == n['name']:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == 'eye_process.started':
                n = {
                    'subject': 'set_detection_mapping_mode',
                    'mode': g_pool.detection_mapping_mode
                }
                ipc_pub.notify(n)
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({
                    'subject': 'meta.doc',
                    'actor': g_pool.app,
                    'doc': world.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            'subject': 'meta.doc',
                            'actor': p.class_name,
                            'doc': p.on_notify.__doc__
                        })

        # window and gl setup
        glfw.glfwInit()
        width, height = session_settings.get('window_size',
                                             (1280 + icon_bar_width, 720))
        main_window = glfw.glfwCreateWindow(width, height,
                                            "Pupil Capture - World")
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = camera_render_size[0] + \
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \
                glfw.glfwGetFramebufferSize(main_window)[1]
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({
                'subject': 'world_process.should_start',
                'delay': 2.
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-400, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos='hidden')
        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
                                             (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.6, .8, 1., 1.2, 1.4],
                        label='Interface size'))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings.append(ui.Button('Reset window size',
                                          set_window_size))
        general_settings.append(
            ui.Selector('audio_mode', audio, selection=audio.audio_modes))
        general_settings.append(
            ui.Selector('detection_mapping_mode',
                        g_pool,
                        label='detection & mapping mode',
                        setter=set_detection_mapping_mode,
                        selection=['2d', '3d']))
        general_settings.append(
            ui.Switch('eye0_process',
                      label='Detect eye 0',
                      setter=lambda alive: start_stop_eye(0, alive),
                      getter=lambda: eyes_are_alive[0].value))
        general_settings.append(
            ui.Switch('eye1_process',
                      label='Detect eye 1',
                      setter=lambda alive: start_stop_eye(1, alive),
                      getter=lambda: eyes_are_alive[1].value))

        general_settings.append(
            ui.Info_Text('Capture Version: {}'.format(g_pool.version)))
        general_settings.append(
            ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed',
                       general_settings,
                       label=chr(0xe8b8),
                       on_val=False,
                       off_val=True,
                       setter=toggle_general_settings,
                       label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(False)

        # now the we have  aproper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        if session_settings.get('eye1_process_alive', False):
            launch_eye_process(1, delay=0.6)
        if session_settings.get('eye0_process_alive', True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({'subject': 'world_process.started'})
        logger.warning('Process started.')

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            #a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # send new events to ipc:
            del events['pupil_positions']  # already on the wire
            del events['gaze_positions']  # sent earlier
            if 'frame' in events:
                del events['frame']  # send explicity with frame publisher
            if 'depth_frame' in events:
                del events['depth_frame']
            if 'audio_packets' in events:
                del events['audio_packets']
            del events['dt']  # no need to send this
            for topic, data in events.items():
                assert (isinstance(data, (list, tuple)))
                for d in data:
                    ipc_pub.send(topic, d)

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if window_should_update() and gl_utils.is_window_visible(
                    main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                unused_elements = g_pool.gui.update()
                for button, action, mods in unused_elements.buttons:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in unused_elements.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in unused_elements.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                glfw.glfwSwapBuffers(main_window)
            glfw.glfwPollEvents()

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings['eye0_process_alive'] = eyes_are_alive[0].value
        session_settings['eye1_process_alive'] = eyes_are_alive[1].value
        session_settings[
            'detection_mapping_mode'] = g_pool.detection_mapping_mode
        session_settings['audio_mode'] = audio.audio_mode
        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Capture crashed with trace:\n{}'.format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'world_process.stopped'})
        sleep(1.0)
Beispiel #20
0
    def __init__(
        self,
        g_pool,
        window_size=window_size_default,
        window_position=window_position_default,
        gui_scale=1.0,
        ui_config={},
    ):
        super().__init__(g_pool)

        self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128

        glfw.glfwInit()
        if g_pool.hide_ui:
            glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0)  # hide window
        main_window = glfw.glfwCreateWindow(*window_size, "Pupil Service")
        glfw.glfwSetWindowPos(main_window, *window_position)
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = gui_scale
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(0, 0),
                                           size=(0, 0),
                                           header_pos="headline")
        g_pool.gui.append(g_pool.menubar)

        # Callback functions
        def on_resize(window, w, h):
            self.window_size = w, h
            self.hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * self.hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * self.hdpi_factor, y * self.hdpi_factor
            g_pool.gui.update_mouse(x, y)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *self.window_size)

        def set_window_size():
            glfw.glfwSetWindowSize(main_window, *window_size_default)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            self.notify_all({"subject": "clear_settings_process.should_start"})
            self.notify_all({
                "subject": "service_process.should_start",
                "delay": 2.0
            })

        g_pool.menubar.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.6, 0.8, 1.0, 1.2, 1.4],
                label="Interface size",
            ))

        g_pool.menubar.append(ui.Button("Reset window size", set_window_size))

        pupil_remote_addr = "{}:{}".format(
            socket.gethostbyname(socket.gethostname()),
            g_pool.preferred_remote_port)
        g_pool.menubar.append(
            ui.Text_Input(
                "pupil_remote_addr",
                getter=lambda: pupil_remote_addr,
                setter=lambda x: None,
                label="Pupil Remote address",
            ))
        g_pool.menubar.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: self.start_stop_eye(0, alive),
                getter=lambda: g_pool.eye_procs_alive[0].value,
            ))
        g_pool.menubar.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: self.start_stop_eye(1, alive),
                getter=lambda: g_pool.eye_procs_alive[1].value,
            ))

        g_pool.menubar.append(
            ui.Info_Text("Service Version: {}".format(g_pool.version)))

        g_pool.menubar.append(
            ui.Button("Restart with default settings", reset_restart))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        g_pool.gui.configuration = ui_config
        gl_utils.basic_gl_setup()

        on_resize(g_pool.main_window,
                  *glfw.glfwGetFramebufferSize(main_window))
Beispiel #21
0
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
                app_version):
    # general imports
    import logging
    # networking
    import zmq
    import zmq_tools
    from time import sleep

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        import glfw
        import gl_utils
        from OpenGL.GL import glClearColor
        from version_utils import VersionFormat
        from file_methods import Persistent_Dict
        from pyglui.pyfontstash import fontstash
        from pyglui.ui import get_roboto_font_path
        from player_methods import is_pupil_rec_dir, update_recording_to_recent

        def on_drop(window, count, paths):
            nonlocal rec_dir
            rec_dir = paths[0].decode('utf-8')

        if rec_dir:
            if not is_pupil_rec_dir(rec_dir):
                rec_dir = None
        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != app_version:
            logger.info(
                "Session setting are from a  different version of this app. I will not use those."
            )
            session_settings.clear()
        w, h = session_settings.get('window_size', (1280, 720))
        window_pos = session_settings.get('window_position',
                                          window_position_default)

        glfw.glfwInit()
        glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0)
        window = glfw.glfwCreateWindow(w, h, 'Pupil Player')
        glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1)

        glfw.glfwMakeContextCurrent(window)
        glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1])
        glfw.glfwSetDropCallback(window, on_drop)

        glfont = fontstash.Context()
        glfont.add_font('roboto', get_roboto_font_path())
        glfont.set_align_string(v_align="center", h_align="middle")
        glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
        gl_utils.basic_gl_setup()
        glClearColor(0.5, .5, 0.5, 0.0)
        text = 'Drop a recording directory onto this window.'
        tip = '(Tip: You can drop a recording directory onto the app icon.)'
        # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
        while not glfw.glfwWindowShouldClose(window):

            fb_size = glfw.glfwGetFramebufferSize(window)
            hdpi_factor = float(fb_size[0] / glfw.glfwGetWindowSize(window)[0])
            gl_utils.adjust_gl_view(*fb_size)

            if rec_dir:
                if is_pupil_rec_dir(rec_dir):
                    logger.info(
                        "Starting new session with '{}'".format(rec_dir))
                    text = "Updating recording format."
                    tip = "This may take a while!"
                else:
                    logger.error(
                        "'{}' is not a valid pupil recording".format(rec_dir))
                    tip = "Oops! That was not a valid recording."
                    rec_dir = None

            gl_utils.clear_gl_screen()
            glfont.set_blur(10.5)
            glfont.set_color_float((0.0, 0.0, 0.0, 1.))
            glfont.set_size(w / 25. * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, .3 * h * hdpi_factor, text)
            glfont.set_size(w / 30. * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, .4 * h * hdpi_factor, tip)
            glfont.set_blur(0.96)
            glfont.set_color_float((1., 1., 1., 1.))
            glfont.set_size(w / 25. * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, .3 * h * hdpi_factor, text)
            glfont.set_size(w / 30. * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, .4 * h * hdpi_factor, tip)

            glfw.glfwSwapBuffers(window)

            if rec_dir:
                update_recording_to_recent(rec_dir)
                glfw.glfwSetWindowShouldClose(window, True)

            glfw.glfwPollEvents()

        session_settings['window_position'] = glfw.glfwGetWindowPos(window)
        session_settings.close()
        glfw.glfwDestroyWindow(window)
        if rec_dir:
            ipc_pub.notify({
                "subject": "player_process.should_start",
                "rec_dir": rec_dir
            })

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error(
            'Process player_drop crashed with trace:\n{}'.format(trace))

    finally:
        sleep(1.0)
Beispiel #22
0
    def __init__(self,
                 g_pool,
                 window_size=window_size_default,
                 window_position=window_position_default,
                 gui_scale=1.,
                 ui_config={}):
        super().__init__(g_pool)

        self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(*window_size, "Pupil Service")
        glfw.glfwSetWindowPos(main_window, *window_position)
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = gui_scale
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(0, 0),
                                           size=(0, 0),
                                           header_pos='headline')
        g_pool.gui.append(g_pool.menubar)

        # Callback functions
        def on_resize(window, w, h):
            self.window_size = w, h
            self.hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * self.hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * self.hdpi_factor, y * self.hdpi_factor
            g_pool.gui.update_mouse(x, y)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *self.window_size)

        def set_window_size():
            glfw.glfwSetWindowSize(main_window, *window_size_default)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            self.notify_all({'subject': 'clear_settings_process.should_start'})
            self.notify_all({
                'subject': 'service_process.should_start',
                'delay': 2.
            })

        g_pool.menubar.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.6, .8, 1., 1.2, 1.4],
                        label='Interface size'))

        g_pool.menubar.append(ui.Button('Reset window size', set_window_size))

        pupil_remote_addr = '{}:50020'.format(
            socket.gethostbyname(socket.gethostname()))
        g_pool.menubar.append(
            ui.Text_Input('pupil_remote_addr',
                          getter=lambda: pupil_remote_addr,
                          setter=lambda x: None,
                          label='Pupil Remote address'))

        g_pool.menubar.append(
            ui.Selector('detection_mapping_mode',
                        g_pool,
                        label='Detection & mapping mode',
                        setter=self.set_detection_mapping_mode,
                        selection=['disabled', '2d', '3d']))
        g_pool.menubar.append(
            ui.Switch('eye0_process',
                      label='Detect eye 0',
                      setter=lambda alive: self.start_stop_eye(0, alive),
                      getter=lambda: g_pool.eyes_are_alive[0].value))
        g_pool.menubar.append(
            ui.Switch('eye1_process',
                      label='Detect eye 1',
                      setter=lambda alive: self.start_stop_eye(1, alive),
                      getter=lambda: g_pool.eyes_are_alive[1].value))

        g_pool.menubar.append(
            ui.Info_Text('Service Version: {}'.format(g_pool.version)))

        g_pool.menubar.append(
            ui.Button('Restart with default settings', reset_restart))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        g_pool.gui.configuration = ui_config
        gl_utils.basic_gl_setup()

        on_resize(g_pool.main_window,
                  *glfw.glfwGetFramebufferSize(main_window))
Beispiel #23
0
def main():

    # Callback functions
    def on_resize(window,w, h):
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
        w,h = w*hdpi_factor, h*hdpi_factor
        g_pool.gui.update_window(w,h)
        g_pool.gui.collect_menus()
        graph.adjust_size(w,h)
        adjust_gl_view(w,h)
        glfwMakeContextCurrent(active_window)
        for p in g_pool.plugins:
            p.on_window_resize(window,w,h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)


    def on_button(window,button, action, mods):
        g_pool.gui.update_button(button,action,mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos,glfwGetWindowSize(main_window))
        pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos,button,action)

    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        x,y = x*hdpi_factor,y*hdpi_factor
        g_pool.gui.update_mouse(x,y)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*y_scroll_factor)


    def on_close(window):
        glfwSetWindowShouldClose(main_window,True)
        logger.debug('Process closing from window')


    try:
        rec_dir = sys.argv[1]
    except:
        #for dev, supply hardcoded dir:
        rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
        if os.path.isdir(rec_dir):
            logger.debug("Dev option: Using hadcoded data dir.")
        else:
            if getattr(sys, 'frozen', False):
                logger.warning("You did not supply a data directory when you called this script! \
                   \nPlease drag a Pupil recoding directory onto the launch icon.")
            else:
                logger.warning("You did not supply a data directory when you called this script! \
                       \nPlease supply a Pupil recoding directory as first arg when calling Pupil Player.")
            return

    if not is_pupil_rec_dir(rec_dir):
        logger.error("You did not supply a dir with the required files inside.")
        return

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))

    #backwards compatibility fn.
    patch_meta_info(rec_dir)

    #parse info.csv file
    meta_info_path = rec_dir + "/info.csv"
    with open(meta_info_path) as info:
        meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) )


    rec_version = read_rec_version(meta_info)
    if rec_version < VersionFormat('0.4'):
        video_path = rec_dir + "/world.avi"
        timestamps_path = rec_dir + "/timestamps.npy"
    else:
        video_path = rec_dir + "/world.mkv"
        timestamps_path = rec_dir + "/world_timestamps.npy"

    gaze_positions_path = rec_dir + "/gaze_positions.npy"
    #load gaze information
    gaze_list = np.load(gaze_positions_path)
    timestamps = np.load(timestamps_path)

    #correlate data
    if rec_version < VersionFormat('0.4'):
        positions_by_frame = correlate_gaze_legacy(gaze_list,timestamps)
    else:
        positions_by_frame = correlate_gaze(gaze_list,timestamps)

    # Initialize capture
    cap = autoCreateCapture(video_path,timestamps=timestamps_path)

    if isinstance(cap,FakeCapture):
        logger.error("could not start capture.")
        return

    width,height = session_settings.get('window_size',cap.get_size())
    window_pos = session_settings.get('window_position',(0,0)) # not yet using this one.


    # Initialize glfw
    glfwInit()
    main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()


    # Register callbacks main_window
    glfwSetWindowSizeCallback(main_window,on_resize)
    glfwSetWindowCloseCallback(main_window,on_close)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)


    # create container for globally scoped vars (within world)
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.version = get_version(version_file)
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.gaze_list = gaze_list
    g_pool.positions_by_frame = positions_by_frame
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.rec_version = rec_version
    g_pool.meta_info = meta_info


    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except FileSeekError:
            pass
        g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index()-2)
        except FileSeekError:
            pass
        g_pool.new_seek = True

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def get_scale():
        return g_pool.gui.scale

    def open_plugin(plugin):
        if plugin ==  "Select to load":
            return
        logger.debug('Open Plugin: %s'%plugin)
        new_plugin = plugin(g_pool)
        g_pool.plugins.add(new_plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive=False
        g_pool.plugins.clean()


    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.main_menu = ui.Scrolling_Menu("Settings",pos=(-350,20),size=(300,300))
    g_pool.main_menu.append(ui.Button("quit",lambda: on_close(None)))
    g_pool.main_menu.configuration = session_settings.get('main_menu_config',{})
    g_pool.main_menu.append(ui.Slider('scale', setter=set_scale,getter=get_scale,step = .05,min=0.75,max=2.5,label='Interface Size'))

    g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version))
    g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version))

    g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
                                        setter= open_plugin, getter = lambda: "Select to load"))
    g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins))
    g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.get_size()[0],cap.get_size()[1])) )


    g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
    g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE)
    g_pool.play_button.on_color[:] = (0,1.,.0,.8)
    g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT)
    g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT)
    g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button])

    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)


    #we always load these plugins
    system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})]
    default_plugins = [('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Export_Launcher',{})]
    previous_plugins = session_settings.get('loaded_plugins',default_plugins)
    g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins)

    for p in g_pool.plugins:
        if p.class_name == 'Trim_Marks':
            g_pool.trim_marks = p
            break

    #set the last saved window size
    on_resize(main_window, *glfwGetWindowSize(main_window))
    glfwSetWindowPos(main_window,0,0)


    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = create_named_texture((height,width,3))

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = cap.get_now()-.03

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,110)
    cpu_graph.update_fn = ps.get_cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260,110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    while not glfwWindowShouldClose(main_window):

        #grab new frame
        if g_pool.play or g_pool.new_seek:
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                #end of video logic: pause at last frame.
                g_pool.play=False

            if g_pool.new_seek:
                display_time = new_frame.timestamp
                g_pool.new_seek = False


            update_graph = True
        else:
            update_graph = False


        frame = new_frame.copy()
        events = {}
        #new positons we make a deepcopy just like the image is a copy.
        events['pupil_positions'] = deepcopy(positions_by_frame[frame.index])

        if update_graph:
            #update performace graphs
            for p in  events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts != t:
                dt,ts = t-ts,t
            fps_graph.add(1./dt)

            g_pool.play_button.status_text = str(frame.index)
        #always update the CPU graph
        cpu_graph.update()


        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        make_coord_system_norm_based()
        update_named_texture(g_pool.image_tex,frame.img)
        draw_named_texture(g_pool.image_tex)
        make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        graph.pop_view()
        g_pool.gui.update()

        #present frames at appropriate speed
        wait_time = frame.timestamp - display_time
        display_time = frame.timestamp
        try:
            spent_time = time()-timestamp
            sleep(wait_time-spent_time)
        except:
            pass
        timestamp = time()


        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['main_menu_config'] = g_pool.main_menu.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)

    session_settings.close()
    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.close()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    logger.debug("Process done")
Beispiel #24
0
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url,ipc_push_url, user_dir, version, eye_id,overwrite_cap_settings=None):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """


    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools
    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=("notify",))

    with Is_Alive_Manager(is_alive_flag,ipc_socket,eye_id):

        #logging setup
        import logging
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logger = logging.getLogger()
        logger.handlers = []
        logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url))
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui,graph,cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
        import OpenGL.GL as gl
        from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based,is_window_visible
        from ui_roi import UIRoi
        #monitoring
        import psutil
        import math


        # helpers/utils
        from uvc import get_time_monotonic, StreamError
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from av_writer import JPEG_Writer,AV_Writer

        from video_capture import InitialisationError,StreamError, Fake_Source,EndofVideoFileError, source_classes, manager_classes
        source_by_name = {src.class_name():src for src in source_classes}

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}



        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600,300*eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600,31+300*eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600,300*eye_id)


        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic()-g_pool.timebase.value
        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window,w, h):
            if is_window_visible(window):
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w,h)
                graph.adjust_size(w,h)
                adjust_gl_view(w,h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key,scancode,action,mods)

        def on_char(window,char):
            g_pool.gui.update_char(char)

        def on_iconify(window,iconified):
            g_pool.iconified = iconified

        def on_button(window,button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1-pos[0],1-pos[1]
                    pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
                        return # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button,action,mods)



        def on_pos(window,x, y):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height) )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)

        def on_scroll(window,x,y):
            g_pool.gui.update_scroll(x,y*scroll_factor)

        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
        if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
            logger.info("Session setting are from older version of this app. I will not use those.")
            session_settings.clear()

        capture_manager_settings = session_settings.get(
            'capture_manager_settings', ('UVC_Manager',{}))

        if eye_id == 0:
            cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
        else:
            cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]

        # Initialize capture
        default_settings = {
            'source_class_name': 'UVC_Source',
            'preferred_names'  : cap_src,
            'frame_size': (640,480),
            'frame_rate': 90
        }
        settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
        try:
            cap = source_by_name[settings['source_class_name']](g_pool, **settings)
        except (KeyError,InitialisationError) as e:
            if isinstance(e,KeyError):
                logger.warning('Incompatible capture setting encountered. Falling back to fake source.')
            cap = Fake_Source(g_pool, **settings)

        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get('flip',False)
        g_pool.display_mode = session_settings.get('display_mode','camera_image')
        g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                    'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
                                    'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}



        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get('roi')
        if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        writer = None

        pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
        last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()


        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]


        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)


        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s"%eye_id
        width,height = session_settings.get('window_size',g_pool.capture.frame_size)
        main_window = glfw.glfwCreateWindow(width,height, title, None, None)
        window_pos = session_settings.get('window_position',window_position_default)
        glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        glfw.glfwSwapInterval(0)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale',1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
        general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
        general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
        general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
        g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
        general_settings.append(detector_selector)

        g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
        g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
        g_pool.capture.init_gui()

        g_pool.sidebar.append(general_settings)
        g_pool.sidebar.append(g_pool.capture_selector_menu)
        g_pool.sidebar.append(g_pool.capture_source_menu)

        g_pool.pupil_detector.init_gui(g_pool.sidebar)

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__:c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
        g_pool.capture_manager.init_gui()

        def open_manager(manager_class):
            g_pool.capture_manager.cleanup()
            g_pool.capture_manager = manager_class(g_pool)
            g_pool.capture_manager.init_gui()

        #We add the capture selection menu, after a manager has been added:
        g_pool.capture_selector_menu.insert(0,ui.Selector(
            'capture_manager',g_pool,
            setter    = open_manager,
            getter    = lambda: g_pool.capture_manager.__class__,
            selection = manager_classes,
            labels    = [b.gui_name for b in manager_classes],
            label     = 'Manager'
        ))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
        glfw.glfwSetKeyCallback(main_window,on_key)
        glfw.glfwSetCharCallback(main_window,on_char)
        glfw.glfwSetMouseButtonCallback(main_window,on_button)
        glfw.glfwSetCursorPosCallback(main_window,on_pos)
        glfw.glfwSetScrollCallback(main_window,on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))


        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config',{})


        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20,130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140,130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"

        should_publish_frames = False
        frame_publish_format = 'jpeg'

        #create a timer to control window update frequency
        window_update_timer = timer(1/60.)
        def window_should_update():
            return next(window_update_timer)

        logger.warning('Process started.')

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t,notification = notify_sub.recv()
                subject = notification['subject']
                if subject == 'eye_process.should_stop':
                    if notification['eye_id'] == eye_id:
                        break
                elif subject == 'set_detection_mapping_mode':
                    if notification['mode'] == '3d':
                        if not isinstance(g_pool.pupil_detector,Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only  = True
                    else:
                        if not isinstance(g_pool.pupil_detector,Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                elif subject == 'recording.started':
                    if notification['record_eye']:
                        record_path = notification['rec_path']
                        raw_mode = notification['compression']
                        logger.info("Will save eye video to: %s"%record_path)
                        timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
                        if raw_mode and frame.jpeg_buffer:
                            video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                            writer = JPEG_Writer(video_path,g_pool.capture.frame_rate)
                        else:
                            video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                            writer = AV_Writer(video_path,g_pool.capture.frame_rate)
                        timestamps = []
                elif subject == 'recording.stopped':
                    if writer:
                        logger.info("Done recording.")
                        writer.release()
                        writer = None
                        np.save(timestamps_path,np.asarray(timestamps))
                        del timestamps
                elif subject.startswith('meta.should_doc'):
                    ipc_socket.notify({
                        'subject':'meta.doc',
                        'actor':'eye%i'%eye_id,
                        'doc':eye.__doc__
                        })
                elif subject.startswith('frame_publishing.started'):
                    should_publish_frames = True
                    frame_publish_format = notification.get('format','jpeg')
                elif subject.startswith('frame_publishing.stopped'):
                    should_publish_frames = False
                    frame_publish_format = 'jpeg'
                else:
                    g_pool.capture_manager.on_notify(notification)

            # Get an image from the grabber
            try:
                frame = g_pool.capture.get_frame()
            except StreamError as e:
                logger.error("Error getting frame. Stopping eye process.")
                logger.debug("Caught error: %s"%e)
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                g_pool.capture.seek_to_frame(0)
                frame = g_pool.capture.get_frame()

            g_pool.u_r = UIRoi((frame.height,frame.width))
            g_pool.capture_manager.update(frame, {})

            if should_publish_frames and frame.jpeg_buffer:
                if   frame_publish_format == "jpeg":
                    data = frame.jpeg_buffer
                elif frame_publish_format == "yuv":
                    data = frame.yuv_buffer
                elif frame_publish_format == "bgr":
                    data = frame.bgr
                elif frame_publish_format == "gray":
                    data = frame.gray
                pupil_socket.send('frame.eye.%s'%eye_id,{
                    'width': frame.width,
                    'height': frame.width,
                    'index': frame.index,
                    'timestamp': frame.timestamp,
                    'format': frame_publish_format,
                    '__raw_data__': [data]
                })


            #update performace graphs
            t = frame.timestamp
            dt,ts = t-ts,t
            try:
                fps_graph.add(1./dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()



            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)


            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            pupil_socket.send('pupil.%s'%eye_id,result)

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image','roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    window_size =  glfw.glfwGetWindowSize(main_window)
                    make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
                    g_pool.capture.gl_display()

                    if result['method'] == '3d c++':

                        eye_ball = result['projected_sphere']
                        try:
                            pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
                                                (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
                                                int(eye_ball['angle']),0,360,8)
                        except ValueError as e:
                            pass
                        else:
                            draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )

                    if result['confidence'] >0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
                                            (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
                                            int(result['ellipse']['angle']),0,360,15)
                            confidence = result['confidence'] * 0.7 #scale it a little
                            draw_polyline(pts,1,RGBA(1.,0,0,confidence))
                            draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize() #detector decides if we visualize or not


        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path,np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window) #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = g_pool.version
        session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.capture.deinit_gui()
        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()
        logger.info("Process shutting down.")
Beispiel #25
0
def world(g_pool, cap_src, cap_size):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    # Callback functions
    def on_resize(window, w, h):
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(
            window)[0]
        w, h = w * hdpi_factor, h * hdpi_factor
        g_pool.gui.update_window(w, h)
        g_pool.gui.collect_menus()
        graph.adjust_size(w, h)
        adjust_gl_view(w, h)
        for p in g_pool.plugins:
            p.on_window_resize(window, w, h)

        glfwMakeContextCurrent(active_window)

    def on_iconify(window, iconfied):
        if not isinstance(cap, FakeCapture):
            g_pool.update_textures = not iconfied

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos, glfwGetWindowSize(main_window))
        pos = denormalize(
            pos,
            (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_world'))

    # Initialize capture
    cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.update_textures = 2

    if isinstance(cap, FakeCapture):
        g_pool.update_textures = 0
    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get(
        'pupil_confidence_threshold', .6)
    g_pool.active_calibration_plugin = None

    #UI callback functions
    def reset_timebase():
        #the last frame from worldcam will be t0
        g_pool.timebase.value = g_pool.capture.get_now()
        logger.info(
            "New timebase set to %s all timestamps will count from here now." %
            g_pool.timebase.value)

    def set_calibration_plugin(new_calibration):
        g_pool.active_calibration_plugin = new_calibration
        new_plugin = new_calibration(g_pool)
        g_pool.plugins.add(new_plugin)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        logger.debug('Open Plugin: %s' % plugin)
        new_plugin = plugin(g_pool)
        g_pool.plugins.add(new_plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def get_scale():
        return g_pool.gui.scale

    width, height = session_settings.get('window_size',
                                         (frame.width, frame.height))
    window_pos = session_settings.get('window_position',
                                      (0, 0))  # not yet using this one.

    # Initialize glfw
    glfwInit()
    main_window = glfwCreateWindow(width, height, "World", None, None)
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    # Register callbacks main_window
    glfwSetWindowSizeCallback(main_window, on_resize)
    glfwSetWindowCloseCallback(main_window, on_close)
    glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = create_named_texture(frame.img.shape)
    update_named_texture(g_pool.image_tex, frame.img)

    # refresh speed settings
    glfwSwapInterval(0)

    glfwSetWindowPos(main_window, 0, 0)

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-250, 0),
                                       size=(0, 0),
                                       header_pos='left')
    g_pool.sidebar.configuration = session_settings.get('side_bar_config', {})
    general_settings = ui.Growing_Menu('General')
    general_settings.configuration = session_settings.get(
        'general_menu_config', {})
    general_settings.append(
        ui.Slider('scale',
                  setter=set_scale,
                  getter=get_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface size'))
    general_settings.append(
        ui.Button(
            'Reset window size',
            lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('Open plugin',
                    selection=user_launchable_plugins,
                    labels=[
                        p.__name__.replace('_', ' ')
                        for p in user_launchable_plugins
                    ],
                    setter=open_plugin,
                    getter=lambda: "Select to load"))
    g_pool.sidebar.append(general_settings)
    advanced_settings = ui.Growing_Menu('Advanced')
    advanced_settings.configuration = session_settings.get(
        'advanced_menu_config', {'collapsed': True})
    advanced_settings.append(
        ui.Selector('update_textures',
                    g_pool,
                    label="Update display",
                    selection=range(3),
                    labels=('No update', 'Gray', 'Color')))
    advanced_settings.append(
        ui.Slider('pupil_confidence_threshold',
                  g_pool,
                  step=.01,
                  min=0.,
                  max=1.,
                  label='Minimum pupil confidence'))
    advanced_settings.append(ui.Button('Set timebase to 0', reset_timebase))
    advanced_settings.append(
        ui.Info_Text('Capture Version: %s' % g_pool.version))

    general_settings.append(advanced_settings)
    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.calibration_menu.configuration = session_settings.get(
        'calibration_menu_config', {})
    g_pool.calibration_menu.append(
        ui.Selector(
            'active_calibration_plugin',
            g_pool,
            selection=calibration_plugins,
            labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
            setter=set_calibration_plugin,
            label='Method'))
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)

    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(
        ui.Hot_Key("quit",
                   setter=on_close,
                   getter=lambda: True,
                   label="X",
                   hotkey=GLFW_KEY_ESCAPE))

    g_pool.capture.init_gui(g_pool.sidebar)
    g_pool.capture.menu.configuration = session_settings.get(
        'capture_menu_config', {})

    #plugins that are loaded based on user settings from previous session
    g_pool.plugins = Plugin_List(
        g_pool, plugin_by_name,
        session_settings.get('loaded_plugins', default_plugins))

    #only needed for the gui to show the loaded calibration type
    for p in g_pool.plugins:
        if p.base_class_name == 'Calibration_Plugin':
            g_pool.active_calibration_plugin = p.__class__
            break

    on_resize(main_window, *glfwGetWindowSize(main_window))

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.get_cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    # Event loop
    while not g_pool.quit.value:

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        fps_graph.add(1. / dt)
        cpu_graph.update()

        #a dictionary that allows plugins to post and read events
        events = {}

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p['confidence'])
        events['pupil_positions'] = recent_pupil_positions

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        if g_pool.update_textures == 2:
            update_named_texture(g_pool.image_tex, frame.img)
        elif g_pool.update_textures == 1:
            update_named_texture(g_pool.image_tex, frame.gray)

        make_coord_system_norm_based()
        draw_named_texture(g_pool.image_tex)
        make_coord_system_pixel_based((frame.height, frame.width, 3))

        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        graph.pop_view()
        g_pool.gui.update()
        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings[
        'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['side_bar_config'] = g_pool.sidebar.configuration
    session_settings['capture_menu_config'] = g_pool.capture.menu.configuration
    session_settings['general_menu_config'] = general_settings.configuration
    session_settings['advanced_menu_config'] = advanced_settings.configuration
    session_settings[
        'calibration_menu_config'] = g_pool.calibration_menu.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.close()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    logger.debug("Process done")
Beispiel #26
0
def world(g_pool,cap_src,cap_size):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from g_pool.pupil_queue
    Can run various plug-ins.
    """


    # Callback functions
    def on_resize(window,w, h):
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        adjust_gl_view(w,h)
        atb.TwWindowSize(w, h)
        glfwMakeContextCurrent(active_window)

    def on_key(window, key, scancode, action, mods):
        if not atb.TwEventKeyboardGLFW(key,action):
            if action == GLFW_PRESS:
                if key == GLFW_KEY_ESCAPE:
                    on_close(window)

    def on_char(window,char):
        if not atb.TwEventCharGLFW(char,1):
            pass

    def on_button(window,button, action, mods):
        if not atb.TwEventMouseButtonGLFW(button,action):
            pos = glfwGetCursorPos(window)
            pos = normalize(pos,glfwGetWindowSize(world_window))
            pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
            for p in g.plugins:
                p.on_click(pos,button,action)

    def on_pos(window,x, y):
        if atb.TwMouseMotion(int(x),int(y)):
            pass

    def on_scroll(window,x,y):
        if not atb.TwMouseWheel(int(x)):
            pass

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')



    # load session persistent settings
    session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_world'),protocol=2)
    def load(var_name,default):
        return session_settings.get(var_name,default)
    def save(var_name,var):
        session_settings[var_name] = var


    # Initialize capture, check if it works
    cap = autoCreateCapture(cap_src, cap_size, 24)
    if cap is None:
        logger.error("Did not receive valid Capture")
        return
    frame = cap.get_frame()
    if frame.img is None:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return
    height,width = frame.img.shape[:2]


    # helpers called by the main atb bar
    def update_fps():
        old_time, bar.timestamp = bar.timestamp, time()
        dt = bar.timestamp - old_time
        if dt:
            bar.fps.value += .05 * (1. / dt - bar.fps.value)

    def set_window_size(mode,data):
        height,width = frame.img.shape[:2]
        ratio = (1,.75,.5,.25)[mode]
        w,h = int(width*ratio),int(height*ratio)
        glfwSetWindowSize(world_window,w,h)
        data.value=mode # update the bar.value

    def get_from_data(data):
        """
        helper for atb getter and setter use
        """
        return data.value

    def open_calibration(selection,data):
        # prepare destruction of current ref_detector... and remove it
        for p in g.plugins:
            if isinstance(p,calibration_routines.detector_by_index):
                p.alive = False
        g.plugins = [p for p in g.plugins if p.alive]

        new_ref_detector = calibration_routines.detector_by_index[selection](g_pool,atb_pos=bar.next_atb_pos)
        g.plugins.append(new_ref_detector)
        g.plugins.sort(key=lambda p: p.order)

        # save the value for atb bar
        data.value=selection

    def toggle_record_video():
        for p in g.plugins:
            if isinstance(p,recorder.Recorder):
                p.alive = False
                return
        # set up folder within recordings named by user input in atb
        if not bar.rec_name.value:
            bar.rec_name.value = recorder.get_auto_name()

        new_plugin = recorder.Recorder(g_pool,bar.rec_name.value, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx)
        g.plugins.append(new_plugin)
        g.plugins.sort(key=lambda p: p.order)

    def toggle_show_calib_result():
        for p in g.plugins:
            if isinstance(p,Show_Calibration):
                p.alive = False
                return

        new_plugin = Show_Calibration(g_pool,frame.img.shape)
        g.plugins.append(new_plugin)
        g.plugins.sort(key=lambda p: p.order)

    def toggle_server():
        for p in g.plugins:
            if isinstance(p,Pupil_Server):
                p.alive = False
                return

        new_plugin = Pupil_Server(g_pool,(10,300))
        g.plugins.append(new_plugin)
        g.plugins.sort(key=lambda p: p.order)


    def toggle_ar():
        for p in g.plugins:
            if isinstance(p,Marker_Detector):
                p.alive = False
                return

        new_plugin = Marker_Detector(g_pool,(10,400))
        g.plugins.append(new_plugin)
        g.plugins.sort(key=lambda p: p.order)

    atb.init()
    # add main controls ATB bar
    bar = atb.Bar(name = "World", label="Controls",
            help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
            text='light', position=(10, 10),refresh=.3, size=(300, 200))
    bar.next_atb_pos = (10,220)
    bar.fps = c_float(0.0)
    bar.timestamp = time()
    bar.calibration_type = c_int(load("calibration_type",0))
    bar.record_eye = c_bool(load("record_eye",0))
    bar.window_size = c_int(load("window_size",0))
    window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
    calibrate_type_enum = atb.enum("Calibration Method",calibration_routines.index_by_name)
    bar.rec_name = create_string_buffer(512)
    bar.version = create_string_buffer(g_pool.version,512)
    bar.rec_name.value = recorder.get_auto_name()
    bar.add_var("fps", bar.fps, step=1., readonly=True)
    bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
    bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.")
    bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.")
    bar.add_var("session name",bar.rec_name, group="Recording", help="creates folder Data_Name_XXX, where xxx is an increasing number")
    bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording")
    bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye")
    bar.add_button("start/stop marker tracking",toggle_ar,key="x",help="find markers in scene to map gaze onto referace surfaces")
    bar.add_button("start/stop server",toggle_server,key="s",help="the server broadcasts pupil and gaze positions locally or via network")
    bar.add_separator("Sep1")
    bar.add_var("version",bar.version, readonly=True)
    bar.add_var("exit", g_pool.quit)

    # add uvc camera controls ATB bar
    cap.create_atb_bar(pos=(320,10))

    # Initialize glfw
    glfwInit()
    world_window = glfwCreateWindow(width, height, "World", None, None)
    glfwMakeContextCurrent(world_window)

    # Register callbacks world_window
    glfwSetWindowSizeCallback(world_window,on_resize)
    glfwSetWindowCloseCallback(world_window,on_close)
    glfwSetKeyCallback(world_window,on_key)
    glfwSetCharCallback(world_window,on_char)
    glfwSetMouseButtonCallback(world_window,on_button)
    glfwSetCursorPosCallback(world_window,on_pos)
    glfwSetScrollCallback(world_window,on_scroll)

    #set the last saved window size
    set_window_size(bar.window_size.value,bar.window_size)
    on_resize(world_window, *glfwGetFramebufferSize(world_window))
    glfwSetWindowPos(world_window,0,0)

    # gl_state settings
    basic_gl_setup()

    # refresh speed settings
    glfwSwapInterval(0)

    # load last calibration data
    try:
        pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy'))
        logger.info("Using calibration found in %s" %g_pool.user_dir)
        map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height))
    except:
        logger.info("No calibration found.")
        def map_pupil(vector):
            """ 1 to 1 mapping
            """
            return vector

    # create container for globally scoped vars (within world)
    g = Temp()
    g.plugins = []
    g_pool.map_pupil = map_pupil

    #load calibration plugin
    open_calibration(bar.calibration_type.value,bar.calibration_type)

    #load gaze_display plugin
    g.plugins.append(Display_Recent_Gaze(g_pool))

    # Event loop
    while not g_pool.quit.value:

        # Get an image from the grabber
        frame = cap.get_frame()
        update_fps()


        #a container that allows plugins to post and read events
        events = []

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            if p['norm_pupil'] is None:
                p['norm_gaze'] = None
            else:
                p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil'])
            recent_pupil_positions.append(p)


        # allow each Plugin to do its work.
        for p in g.plugins:
            p.update(frame,recent_pupil_positions,events)

        #check if a plugin need to be destroyed
        g.plugins = [p for p in g.plugins if p.alive]

        # render camera image
        glfwMakeContextCurrent(world_window)
        draw_gl_texture(frame.img)

        # render visual feedback from loaded plugins
        for p in g.plugins:
            p.gl_display()

        atb.draw()
        glfwSwapBuffers(world_window)
        glfwPollEvents()


    # de-init all running plugins
    for p in g.plugins:
        p.alive = False
        #reading p.alive actually runs plug-in cleanup
        _ = p.alive

    save('window_size',bar.window_size.value)
    save('calibration_type',bar.calibration_type.value)
    save('record_eye',bar.record_eye.value)
    session_settings.close()

    cap.close()
    glfwDestroyWindow(world_window)
    glfwTerminate()
    logger.debug("Process done")
Beispiel #27
0
def eye(g_pool, cap_src, cap_size):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye.log'),
                             mode='w')
    fh.setLevel(logging.INFO)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logging.WARNING)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        'EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter(
        'E Y E Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    # Callback functions
    def on_resize(window, w, h):
        adjust_gl_view(w, h)
        atb.TwWindowSize(w, h)

    def on_key(window, key, scancode, action, mods):
        if not atb.TwEventKeyboardGLFW(key, int(action == GLFW_PRESS)):
            if action == GLFW_PRESS:
                if key == GLFW_KEY_ESCAPE:
                    on_close(window)

    def on_char(window, char):
        if not atb.TwEventCharGLFW(char, 1):
            pass

    def on_button(window, button, action, mods):
        if not atb.TwEventMouseButtonGLFW(button, int(action == GLFW_PRESS)):
            if action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos, glfwGetWindowSize(window))
                pos = denormalize(
                    pos, (frame.img.shape[1],
                          frame.img.shape[0]))  # pos in frame.img pixels
                u_r.setStart(pos)
                bar.draw_roi.value = 1
            else:
                bar.draw_roi.value = 0

    def on_pos(window, x, y):
        if atb.TwMouseMotion(int(x), int(y)):
            pass
        if bar.draw_roi.value == 1:
            pos = x, y
            pos = normalize(pos, glfwGetWindowSize(window))
            pos = denormalize(pos,
                              (frame.img.shape[1],
                               frame.img.shape[0]))  # pos in frame.img pixels
            u_r.setEnd(pos)

    def on_scroll(window, x, y):
        if not atb.TwMouseWheel(int(x)):
            pass

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')

    # Helper functions called by the main atb bar
    def start_roi():
        bar.display.value = 1
        bar.draw_roi.value = 2

    def update_fps():
        old_time, bar.timestamp = bar.timestamp, time()
        dt = bar.timestamp - old_time
        if dt:
            bar.fps.value += .05 * (1. / dt - bar.fps.value)
            bar.dt.value = dt

    def get_from_data(data):
        """
        helper for atb getter and setter use
        """
        return data.value

    # load session persistent settings
    session_settings = shelve.open(os.path.join(g_pool.user_dir,
                                                'user_settings_eye'),
                                   protocol=2)

    def load(var_name, default):
        return session_settings.get(var_name, default)

    def save(var_name, var):
        session_settings[var_name] = var

    # Initialize capture
    cap = autoCreateCapture(cap_src, cap_size)

    if cap is None:
        logger.error("Did not receive valid Capture")
        return
    # check if it works
    frame = cap.get_frame()
    if frame.img is None:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return
    height, width = frame.img.shape[:2]
    cap.auto_rewind = False

    u_r = Roi(frame.img.shape)
    u_r.set(load('roi', default=None))

    writer = None

    pupil_detector = Canny_Detector(g_pool)

    atb.init()
    # Create main ATB Controls
    bar = atb.Bar(name="Eye",
                  label="Display",
                  help="Scene controls",
                  color=(50, 50, 50),
                  alpha=100,
                  text='light',
                  position=(10, 10),
                  refresh=.3,
                  size=(200, 100))
    bar.fps = c_float(0.0)
    bar.timestamp = time()
    bar.dt = c_float(0.0)
    bar.sleep = c_float(0.0)
    bar.display = c_int(load('bar.display', 0))
    bar.draw_pupil = c_bool(load('bar.draw_pupil', True))
    bar.draw_roi = c_int(0)

    dispay_mode_enum = atb.enum("Mode", {
        "Camera Image": 0,
        "Region of Interest": 1,
        "Algorithm": 2
    })

    bar.add_var("FPS", bar.fps, step=1., readonly=True)
    bar.add_var("Mode",
                bar.display,
                vtype=dispay_mode_enum,
                help="select the view-mode")
    bar.add_var("Show_Pupil_Point", bar.draw_pupil)
    bar.add_button("Draw_ROI",
                   start_roi,
                   help="drag on screen to select a region of interest")

    bar.add_var("SlowDown", bar.sleep, step=0.01, min=0.0)
    bar.add_var("SaveSettings&Exit", g_pool.quit)

    cap.create_atb_bar(pos=(220, 10))

    # create a bar for the detector
    pupil_detector.create_atb_bar(pos=(10, 120))

    glfwInit()
    window = glfwCreateWindow(width, height, "Eye", None, None)
    glfwMakeContextCurrent(window)

    # Register callbacks window
    glfwSetWindowSizeCallback(window, on_resize)
    glfwSetWindowCloseCallback(window, on_close)
    glfwSetKeyCallback(window, on_key)
    glfwSetCharCallback(window, on_char)
    glfwSetMouseButtonCallback(window, on_button)
    glfwSetCursorPosCallback(window, on_pos)
    glfwSetScrollCallback(window, on_scroll)

    glfwSetWindowPos(window, 800, 0)
    on_resize(window, width, height)

    # gl_state settings
    basic_gl_setup()

    # refresh speed settings
    glfwSwapInterval(0)

    # event loop
    while not g_pool.quit.value:
        frame = cap.get_frame()
        if frame.img is None:
            break
        update_fps()
        sleep(bar.sleep.value)  # for debugging only

        if pupil_detector.should_sleep:
            sleep(16)
            pupil_detector.should_sleep = False

        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if g_pool.eye_rx.poll():
            command = g_pool.eye_rx.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %(record_path)s")
                video_path = os.path.join(record_path, "eye.avi")
                timestamps_path = os.path.join(record_path,
                                               "eye_timestamps.npy")
                writer = cv2.VideoWriter(
                    video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value,
                    (frame.img.shape[1], frame.img.shape[0]))
                timestamps = []
            else:
                logger.info("Done recording eye.")
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write(frame.img)
            timestamps.append(frame.timestamp)

        # pupil ellipse detection
        result = pupil_detector.detect(frame,
                                       user_roi=u_r,
                                       visualize=bar.display.value == 2)
        # stream the result
        g_pool.pupil_queue.put(result)

        # VISUALIZATION direct visualizations on the frame.img data
        if bar.display.value == 1:
            # and a solid (white) frame around the user defined ROI
            r_img = frame.img[u_r.lY:u_r.uY, u_r.lX:u_r.uX]
            r_img[:, 0] = 255, 255, 255
            r_img[:, -1] = 255, 255, 255
            r_img[0, :] = 255, 255, 255
            r_img[-1, :] = 255, 255, 255

        # GL-drawing
        clear_gl_screen()
        draw_gl_texture(frame.img)

        if result['norm_pupil'] is not None and bar.draw_pupil.value:
            if result.has_key('axes'):
                pts = cv2.ellipse2Poly(
                    (int(result['center'][0]), int(result['center'][1])),
                    (int(result["axes"][0] / 2), int(result["axes"][1] / 2)),
                    int(result["angle"]), 0, 360, 15)
                draw_gl_polyline(pts, (1., 0, 0, .5))
            draw_gl_point_norm(result['norm_pupil'], color=(1., 0., 0., 0.5))

        atb.draw()
        glfwSwapBuffers(window)
        glfwPollEvents()

    # END while running

    # in case eye reconding was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path, np.asarray(timestamps))

    # save session persistent settings
    save('roi', u_r.get())
    save('bar.display', bar.display.value)
    save('bar.draw_pupil', bar.draw_pupil.value)
    session_settings.close()

    pupil_detector.cleanup()
    cap.close()
    atb.terminate()
    glfwDestroyWindow(window)
    glfwTerminate()

    #flushing queue incase world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")
Beispiel #28
0
def world(g_pool,cap_src,cap_size):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    #manage plugins
    runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
    user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins
    system_plugins  = [Log_Display,Display_Recent_Gaze,Recorder]
    plugin_by_index =  system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index,plugin_by_index))
    default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]



    # Callback functions
    def on_resize(window,w, h):
        if not g_pool.iconified:
            g_pool.gui.update_window(w,h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w,h)
            adjust_gl_view(w,h)
            for p in g_pool.plugins:
                p.on_window_resize(window,w,h)

    def on_iconify(window,iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)

    def on_button(window,button, action, mods):
        g_pool.gui.update_button(button,action,mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos,glfwGetWindowSize(main_window))
        pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos,button,action)

    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        x,y = x*hdpi_factor,y*hdpi_factor
        g_pool.gui.update_mouse(x,y)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*scroll_factor)


    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')


    tick = delta_t()
    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
    if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size':cap_size,'frame_rate':24}
    previous_settings = session_settings.get('capture_settings',None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return


    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.iconified = False

    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
    g_pool.active_calibration_plugin = None


    def open_plugin(plugin):
        if plugin ==  "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()


    #window and gl setup
    glfwInit()
    width,height = session_settings.get('window_size',(frame.width, frame.height))
    main_window = glfwCreateWindow(width,height, "World")
    window_pos = session_settings.get('window_position',window_position_default)
    glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
    general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
    general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
                                        setter= open_plugin, getter=lambda: "Select to load"))
    general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
    general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
    g_pool.sidebar.append(general_settings)

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.capture.init_gui(g_pool.sidebar)

    #plugins that are loaded based on user settings from previous session
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
                                        setter= open_plugin,label='Method'))

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window,on_resize)
    glfwSetWindowCloseCallback(main_window,on_close)
    glfwSetWindowIconifyCallback(main_window,on_iconify)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)

    # refresh speed settings
    glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config',{})



    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260,130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    # Event loop
    while not g_pool.quit.value:

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt,ts = t-ts,t
        try:
            fps_graph.add(1./dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()


        #a dictionary that allows plugins to post and read events
        events = {}

        #report time between now and the last loop interation
        events['dt'] = get_dt()

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p['confidence'])
        events['pupil_positions'] = recent_pupil_positions


        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifications:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        if g_pool.iconified:
            pass
        else:
            g_pool.image_tex.update_from_frame(frame)

        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height,frame.width,3))
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        if not g_pool.iconified:
            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfwSwapBuffers(main_window)
        glfwPollEvents()

    glfwRestoreWindow(main_window) #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    logger.debug("Process done")
Beispiel #29
0
def world(
    timebase,
    eye_procs_alive,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    preferred_remote_port,
    hide_ui,
    debug,
):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``eye_process.started``
        ``start_plugin``
        ``should_stop``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            "subject": "eye_process.should_start.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": delay,
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            "subject": "eye_process.should_stop.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": 0.2,
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def detection_enabled_getter() -> bool:
        return g_pool.pupil_detection_enabled

    def detection_enabled_setter(is_on: bool):
        g_pool.pupil_detection_enabled = is_on
        n = {"subject": "set_pupil_detection_enabled", "value": is_on}
        ipc_pub.notify(n)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        from OpenGL.GL import GL_COLOR_BUFFER_BIT

        # display
        import glfw

        glfw.ERROR_REPORTING = "raise"

        from version_utils import parse_version
        from pyglui import ui, cygl, __version__ as pyglui_version

        assert parse_version(pyglui_version) >= parse_version(
            "1.27"
        ), "pyglui out of date, please upgrade to newest version"
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic

        logger.info("Application Version: {}".format(version))
        logger.info("System Info: {}".format(get_system_info()))
        logger.debug(f"Debug flag: {debug}")

        import audio

        # Plug-ins
        from plugin import (
            Plugin,
            System_Plugin_Base,
            Plugin_List,
            import_runtime_plugins,
        )
        from plugin_manager import Plugin_Manager
        from calibration_choreography import (
            available_calibration_choreography_plugins,
            CalibrationChoreographyPlugin,
            patch_loaded_plugins_with_choreography_plugin,
        )

        available_choreography_plugins = available_calibration_choreography_plugins()

        from gaze_mapping import registered_gazer_classes
        from gaze_mapping.gazer_base import GazerBase
        from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from network_api import NetworkApiPlugin
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker_Online
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from blink_detection import Blink_Detection
        from video_capture import (
            source_classes,
            manager_classes,
            Base_Manager,
            Base_Source,
        )
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from accuracy_visualizer import Accuracy_Visualizer

        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay
        from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (8, 90)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        content_scale = 1.0

        # g_pool holds variables for this process they are accessible to all plugins
        g_pool = SimpleNamespace()
        g_pool.debug = debug
        g_pool.app = "capture"
        g_pool.process = "world"
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eye_procs_alive = eye_procs_alive
        g_pool.preferred_remote_port = preferred_remote_port

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, "plugins")
        )
        runtime_plugins = [
            p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin)
        ]
        user_plugins = [
            Pupil_Groups,
            NetworkApiPlugin,
            Time_Sync,
            Surface_Tracker_Online,
            Annotation_Capture,
            Log_History,
            Fixation_Detector,
            Blink_Detection,
            Remote_Recorder,
            Accuracy_Visualizer,
            Camera_Intrinsics_Estimation,
            Hololens_Relay,
            Online_Head_Pose_Tracker,
        ]

        system_plugins = (
            [
                Log_Display,
                Display_Recent_Gaze,
                Recorder,
                Pupil_Data_Relay,
                Plugin_Manager,
                System_Graphs,
            ]
            + manager_classes
            + source_classes
        )
        plugins = (
            system_plugins
            + user_plugins
            + runtime_plugins
            + available_choreography_plugins
            + registered_gazer_classes()
        )
        user_plugins += [
            p
            for p in runtime_plugins
            if not isinstance(
                p,
                (
                    Base_Manager,
                    Base_Source,
                    System_Plugin_Base,
                    CalibrationChoreographyPlugin,
                    GazerBase,
                ),
            )
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_name = "UVC_Source"
        default_capture_settings = {
            "preferred_names": [
                "Pupil Cam1 ID2",
                "Logitech Camera",
                "(046d:081d)",
                "C510",
                "B525",
                "C525",
                "C615",
                "C920",
                "C930e",
            ],
            "frame_size": (1280, 720),
            "frame_rate": 30,
        }

        default_plugins = [
            (default_capture_name, default_capture_settings),
            ("Pupil_Data_Relay", {}),
            ("UVC_Manager", {}),
            ("NDSI_Manager", {}),
            ("HMD_Streaming_Manager", {}),
            ("File_Manager", {}),
            ("Log_Display", {}),
            ("Dummy_Gaze_Mapper", {}),
            ("Display_Recent_Gaze", {}),
            # Calibration choreography plugin is added below by calling
            # patch_loaded_plugins_with_choreography_plugin
            ("Recorder", {}),
            ("NetworkApiPlugin", {}),
            ("Fixation_Detector", {}),
            ("Blink_Detection", {}),
            ("Accuracy_Visualizer", {}),
            ("Plugin_Manager", {}),
            ("System_Graphs", {}),
        ]

        def consume_events_and_render_buffer():
            gl_utils.glViewport(0, 0, *camera_render_size)
            for p in g_pool.plugins:
                p.gl_display()

            gl_utils.glViewport(0, 0, *window_size)
            try:
                clipboard = glfw.get_clipboard_string(main_window).decode()
            except (AttributeError, glfw.GLFWError):
                # clipboard is None, might happen on startup
                clipboard = ""
            g_pool.gui.update_clipboard(clipboard)
            user_input = g_pool.gui.update()
            if user_input.clipboard != clipboard:
                # only write to clipboard if content changed
                glfw.set_clipboard_string(main_window, user_input.clipboard)

            for button, action, mods in user_input.buttons:
                x, y = glfw.get_cursor_pos(main_window)
                pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                    main_window, x, y, cached_scale=None
                )
                pos = normalize(pos, camera_render_size)
                # Position in img pixels
                pos = denormalize(pos, g_pool.capture.frame_size)

                for plugin in g_pool.plugins:
                    if plugin.on_click(pos, button, action):
                        break

            for key, scancode, action, mods in user_input.keys:
                for plugin in g_pool.plugins:
                    if plugin.on_key(key, scancode, action, mods):
                        break

            for char_ in user_input.chars:
                for plugin in g_pool.plugins:
                    if plugin.on_char(char_):
                        break

            glfw.swap_buffers(main_window)

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal content_scale
            if w == 0 or h == 0:
                return

            # Always clear buffers on resize to make sure that there are no overlapping
            # artifacts from previous frames.
            gl_utils.glClear(GL_COLOR_BUFFER_BIT)
            gl_utils.glClearColor(0, 0, 0, 1)

            content_scale = gl_utils.get_content_scale(window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(window)
            g_pool.gui.scale = content_scale
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()

            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

            # Minimum window size required, otherwise parts of the UI can cause openGL
            # issues with permanent effects. Depends on the content scale, which can
            # potentially be dynamically modified, so we re-adjust the size limits every
            # time here.
            min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale)
            glfw.set_window_size_limits(
                window,
                min_size,
                min_size,
                glfw.DONT_CARE,
                glfw.DONT_CARE,
            )

            # Needed, to update the window buffer while resizing
            consume_events_and_render_buffer()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
                window, x, y, cached_scale=None
            )
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, paths):
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_world")
        )
        if parse_version(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.min_data_confidence = 0.6
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", 0.8
        )
        g_pool.pupil_detection_enabled = session_settings.get(
            "pupil_detection_enabled", True
        )
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.set_audio_mode(
            session_settings.get("audio_mode", audio.get_default_audio_mode())
        )

        def handle_notifications(noti):
            subject = noti["subject"]
            if subject == "set_pupil_detection_enabled":
                g_pool.pupil_detection_enabled = noti["value"]
            elif subject == "start_plugin":
                try:
                    g_pool.plugins.add(
                        g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})
                    )
                except KeyError as err:
                    logger.error(f"Attempt to load unknown plugin: {err}")
            elif subject == "stop_plugin":
                for p in g_pool.plugins:
                    if p.class_name == noti["name"]:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == "eye_process.started":
                noti = {
                    "subject": "set_pupil_detection_enabled",
                    "value": g_pool.pupil_detection_enabled,
                }
                ipc_pub.notify(noti)
            elif subject == "set_min_calibration_confidence":
                g_pool.min_calibration_confidence = noti["value"]
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify(
                    {"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__}
                )
                for p in g_pool.plugins:
                    if (
                        p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify
                    ):
                        ipc_pub.notify(
                            {
                                "subject": "meta.doc",
                                "actor": p.class_name,
                                "doc": p.on_notify.__doc__,
                            }
                        )
            elif subject == "world_process.adapt_window_size":
                set_window_size()
            elif subject == "world_process.should_stop":
                glfw.set_window_should_close(main_window, True)

        width, height = session_settings.get(
            "window_size", (1280 + icon_bar_width, 720)
        )

        # window and gl setup
        glfw.init()
        glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
        if hide_ui:
            glfw.window_hint(glfw.VISIBLE, 0)  # hide window
        main_window = glfw.create_window(
            width, height, "Pupil Capture - World", None, None
        )

        window_position_manager = gl_utils.WindowPositionManager()
        window_pos = window_position_manager.new_window_position(
            window=main_window,
            default_position=window_position_default,
            previous_position=session_settings.get("window_position", None),
        )
        glfw.set_window_pos(main_window, window_pos[0], window_pos[1])

        glfw.make_context_current(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.set_window_should_close(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0})

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is opened, the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")

        def set_window_size():
            # Get current capture frame size
            f_width, f_height = g_pool.capture.frame_size

            # Get current display scale factor
            content_scale = gl_utils.get_content_scale(main_window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
            display_scale_factor = content_scale / framebuffer_scale

            # Scale the capture frame size by display scale factor
            f_width *= display_scale_factor
            f_height *= display_scale_factor

            # Increas the width to account for the added scaled icon bar width
            f_width += icon_bar_width * display_scale_factor

            # Set the newly calculated size (scaled capture frame size + scaled icon bar width)
            glfw.set_window_size(main_window, int(f_width), int(f_height))

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(
            ui.Selector(
                "Audio mode",
                None,
                getter=audio.get_audio_mode,
                setter=audio.set_audio_mode,
                selection=audio.get_audio_mode_list(),
            )
        )

        general_settings.append(
            ui.Switch(
                "pupil_detection_enabled",
                label="Pupil detection",
                getter=detection_enabled_getter,
                setter=detection_enabled_setter,
            )
        )
        general_settings.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: start_stop_eye(0, alive),
                getter=lambda: eye_procs_alive[0].value,
            )
        )
        general_settings.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: start_stop_eye(1, alive),
                getter=lambda: eye_procs_alive[1].value,
            )
        )

        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(g_pool.version))
        )
        general_settings.append(
            ui.Button("Restart with default settings", reset_restart)
        )

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        loaded_plugins = session_settings.get("loaded_plugins", default_plugins)

        # Resolve the active calibration choreography plugin
        loaded_plugins = patch_loaded_plugins_with_choreography_plugin(
            loaded_plugins, app=g_pool.app
        )
        session_settings["loaded_plugins"] = loaded_plugins

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(g_pool, loaded_plugins)

        if not g_pool.capture:
            # Make sure we always have a capture running. Important if there was no
            # capture stored in session settings.
            g_pool.plugins.add(
                g_pool.plugin_by_name[default_capture_name], default_capture_settings
            )

        # Register callbacks main_window
        glfw.set_framebuffer_size_callback(main_window, on_resize)
        glfw.set_key_callback(main_window, on_window_key)
        glfw.set_char_callback(main_window, on_window_char)
        glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
        glfw.set_cursor_pos_callback(main_window, on_pos)
        glfw.set_scroll_callback(main_window, on_scroll)
        glfw.set_drop_callback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(True)

        # now that we have a proper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})
        # If previously selected plugin was not loaded this time, we will have an
        # expanded menubar without any menu selected. We need to ensure the menubar is
        # collapsed in this case.
        if all(submenu.collapsed for submenu in g_pool.menubar.elements):
            g_pool.menubar.collapsed = True

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.get_framebuffer_size(main_window))

        if session_settings.get("eye1_process_alive", True):
            launch_eye_process(1, delay=0.6)
        if session_settings.get("eye0_process_alive", True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({"subject": "world_process.started"})
        logger.warning("Process started.")

        if platform.system() == "Darwin":
            # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of
            # occluded windows, based on the swap interval value. This causes an FPS drop
            # and leads to problems when recording. To side-step this behaviour, the swap
            # interval is set to zero.
            #
            # Read more about window occlusion on macOS here:
            # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html
            glfw.swap_interval(0)

        # Event loop
        while not glfw.window_should_close(main_window) and not process_was_interrupted:

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # "blacklisted" events that were already sent
            del events["pupil"]
            del events["gaze"]
            # delete if exists. More expensive than del, so only use it when key might not exist
            events.pop("annotation", None)

            # send new events to ipc:
            if "frame" in events:
                del events["frame"]  # send explicitly with frame publisher
            if "depth_frame" in events:
                del events["depth_frame"]
            if "audio_packets" in events:
                del events["audio_packets"]
            del events["dt"]  # no need to send this
            for data in events.values():
                assert isinstance(data, (list, tuple))
                for d in data:
                    ipc_pub.send(d)

            glfw.make_context_current(main_window)
            # render visual feedback from loaded plugins
            glfw.poll_events()
            if window_should_update() and gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                try:
                    clipboard = glfw.get_clipboard_string(main_window).decode()
                except (AttributeError, glfw.GLFWError):
                    # clipboard is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.set_clipboard_string(main_window, user_input.clipboard)

                for button, action, mods in user_input.buttons:
                    x, y = glfw.get_cursor_pos(main_window)
                    pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                        main_window, x, y, cached_scale=None
                    )
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                glfw.swap_buffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)
        session_settings["eye0_process_alive"] = eye_procs_alive[0].value
        session_settings["eye1_process_alive"] = eye_procs_alive[1].value
        session_settings[
            "min_calibration_confidence"
        ] = g_pool.min_calibration_confidence
        session_settings["pupil_detection_enabled"] = g_pool.pupil_detection_enabled
        session_settings["audio_mode"] = audio.get_audio_mode()

        if not hide_ui:
            glfw.restore_window(main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.get_window_pos(main_window)
            session_window_size = glfw.get_window_size(main_window)
            if 0 not in session_window_size:
                f_width, f_height = session_window_size
                if platform.system() in ("Windows", "Linux"):
                    f_width, f_height = (
                        f_width / content_scale,
                        f_height / content_scale,
                    )
                session_settings["window_size"] = int(f_width), int(f_height)

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.destroy_window(main_window)
        glfw.terminate()

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Capture crashed with trace:\n{}".format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "world_process.stopped"})
        sleep(1.0)
Beispiel #30
0
    def open_window(self, title="new_window"):
        if not self._window:
            if self.fullscreen:
                try:
                    monitor = glfwGetMonitors()[self.monitor_idx]
                except Exception:
                    logger.warning(
                        "Monitor at index %s no longer availalbe using default"
                        % self.monitor_idx)
                    self.monitor_idx = 0
                    monitor = glfwGetMonitors()[self.monitor_idx]
                (
                    width,
                    height,
                    redBits,
                    blueBits,
                    greenBits,
                    refreshRate,
                ) = glfwGetVideoMode(monitor)
            else:
                monitor = None
                width, height = 640, 360

            # NOTE: Always creating windowed window here, even if in fullscreen mode. On
            # windows you might experience a black screen for up to 1 sec when creating
            # a blank window directly in fullscreen mode. By creating it windowed and
            # then switching to fullscreen it will stay white the entire time.
            self._window = glfwCreateWindow(width,
                                            height,
                                            title,
                                            share=glfwGetCurrentContext())

            if not self.fullscreen:
                glfwSetWindowPos(
                    self._window,
                    self.window_position_default[0],
                    self.window_position_default[1],
                )

            # This makes it harder to accidentally tab out of fullscreen by clicking on
            # some other window (e.g. when having two monitors). On the other hand you
            # want a cursor to adjust the window size when not in fullscreen mode.
            cursor = GLFW_CURSOR_DISABLED if self.fullscreen else GLFW_CURSOR_HIDDEN

            glfwSetInputMode(self._window, GLFW_CURSOR, cursor)

            # Register callbacks
            glfwSetFramebufferSizeCallback(self._window, on_resize)
            glfwSetKeyCallback(self._window, self.on_window_key)
            glfwSetMouseButtonCallback(self._window,
                                       self.on_window_mouse_button)
            on_resize(self._window, *glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)

            if self.fullscreen:
                # Switch to full screen here. See NOTE above at glfwCreateWindow().
                glfwSetWindowMonitor(self._window, monitor, 0, 0, width,
                                     height, refreshRate)
Beispiel #31
0
def world(g_pool,cap_src,cap_size):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    #manage plugins
    runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
    user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins
    system_plugins  = [Log_Display,Display_Recent_Gaze,Recorder]
    plugin_by_index =  system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index,plugin_by_index))
    default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]



    # Callback functions
    def on_resize(window,w, h):
        if not g_pool.iconified:
            g_pool.gui.update_window(w,h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w,h)
            adjust_gl_view(w,h)
            for p in g_pool.plugins:
                p.on_window_resize(window,w,h)

    def on_iconify(window,iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)

    def on_button(window,button, action, mods):
        g_pool.gui.update_button(button,action,mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos,glfwGetWindowSize(main_window))
        pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos,button,action)

    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        x,y = x*hdpi_factor,y*hdpi_factor
        g_pool.gui.update_mouse(x,y)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*scroll_factor)


    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')


    tick = delta_t()
    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
    if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size':cap_size,'frame_rate':24}
    previous_settings = session_settings.get('capture_settings',None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return


    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.update_textures = session_settings.get("update_textures",2)
    g_pool.iconified = False

    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
    g_pool.active_calibration_plugin = None


    def open_plugin(plugin):
        if plugin ==  "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()


    #window and gl setup
    glfwInit()
    width,height = session_settings.get('window_size',(frame.width, frame.height))
    main_window = glfwCreateWindow(width,height, "World")
    window_pos = session_settings.get('window_position',window_position_default)
    glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
    general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
    general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
                                        setter= open_plugin, getter=lambda: "Select to load"))
    g_pool.sidebar.append(general_settings)
    advanced_settings = ui.Growing_Menu('Advanced')
    advanced_settings.append(ui.Selector('update_textures',g_pool,label="Update display",selection=range(3),labels=('No update','Gray','Color')))
    advanced_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
    advanced_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
    general_settings.append(advanced_settings)
    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.calibration_menu.append(ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
                                        setter= open_plugin,label='Method'))
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
    g_pool.capture.init_gui(g_pool.sidebar)

    #plugins that are loaded based on user settings from previous session
    g_pool.notifications = []
    g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))


    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window,on_resize)
    glfwSetWindowCloseCallback(main_window,on_close)
    glfwSetWindowIconifyCallback(main_window,on_iconify)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = create_named_texture(frame.img.shape)
    update_named_texture(g_pool.image_tex,frame.img)

    # refresh speed settings
    glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config',{})



    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260,130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    # Event loop
    while not g_pool.quit.value:

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt,ts = t-ts,t
        try:
            fps_graph.add(1./dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()


        #a dictionary that allows plugins to post and read events
        events = {}

        #report time between now and the last loop interation
        events['dt'] = get_dt()

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p['confidence'])
        events['pupil_positions'] = recent_pupil_positions

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        if g_pool.iconified:
            pass
        elif g_pool.update_textures == 2:
            update_named_texture(g_pool.image_tex,frame.img)
        elif g_pool.update_textures == 1:
            update_named_texture(g_pool.image_tex,frame.gray)

        make_coord_system_norm_based()
        draw_named_texture(g_pool.image_tex)

        make_coord_system_pixel_based((frame.height,frame.width,3))
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        if not g_pool.iconified:
            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfwSwapBuffers(main_window)
        glfwPollEvents()

    glfwRestoreWindow(main_window) #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['update_textures'] = g_pool.update_textures
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    logger.debug("Process done")
Beispiel #32
0
def show_no_rec_window():
    from pyglui.pyfontstash import fontstash
    from pyglui.ui import get_roboto_font_path

    def on_drop(window,count,paths):
        for x in range(count):
            new_rec_dir =  paths[x]
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '%s'"%new_rec_dir)
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window,True)
            else:
                logger.error("'%s' is not a valid pupil recording"%new_rec_dir)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
    if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file):
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()
    w,h = session_settings.get('window_size',(1280,720))
    window_pos = session_settings.get('window_position',(0,0))

    glfwWindowHint(GLFW_RESIZABLE,0)
    window = glfwCreateWindow(w, h,'Pupil Player')
    glfwWindowHint(GLFW_RESIZABLE,1)

    glfwMakeContextCurrent(window)
    glfwSetWindowPos(window,window_pos[0],window_pos[1])
    glfwSetDropCallback(window,on_drop)

    adjust_gl_view(w,h)
    glfont = fontstash.Context()
    glfont.add_font('roboto',get_roboto_font_path())
    glfont.set_align_string(v_align="center",h_align="middle")
    glfont.set_color_float((0.2,0.2,0.2,0.9))
    basic_gl_setup()
    glClearColor(0.5,.5,0.5,0.0)
    text = 'Drop a recording directory onto this window.'
    tip = '(Tip: You can drop a recording directory onto the app icon.)'
    # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
    while not glfwWindowShouldClose(window):
        clear_gl_screen()
        glfont.set_blur(10.5)
        glfont.set_color_float((0.0,0.0,0.0,1.))
        glfont.set_size(w/25.)
        glfont.draw_text(w/2,.3*h,text)
        glfont.set_size(w/30.)
        glfont.draw_text(w/2,.4*h,tip)
        glfont.set_blur(0.96)
        glfont.set_color_float((1.,1.,1.,1.))
        glfont.set_size(w/25.)
        glfont.draw_text(w/2,.3*h,text)
        glfont.set_size(w/30.)
        glfont.draw_text(w/2,.4*h,tip)
        glfwSwapBuffers(window)
        glfwPollEvents()

    session_settings['window_position'] = glfwGetWindowPos(window)
    session_settings['version'] = get_version(version_file)
    session_settings.close()
    del glfont
    glfwDestroyWindow(window)
Beispiel #33
0
def world(pupil_queue, timebase, lauchner_pipe, eye_pipes, eyes_are_alive, user_dir, version, cap_src):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    import logging

    # Set up root logger for this process before doing imports of logged modules.
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(user_dir, "world.log"), mode="w")
    fh.setLevel(logger.level)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level + 10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter("World Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s")
    fh.setFormatter(formatter)
    formatter = logging.Formatter("WORLD Process [%(levelname)s] %(name)s : %(message)s")
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # silence noisy modules
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logging.getLogger("libav").setLevel(logging.ERROR)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmfull but unnessasary.

    # general imports
    from time import time
    import numpy as np

    # display
    import glfw
    from pyglui import ui, graph, cygl
    from pyglui.cygl.utils import Named_Texture
    from gl_utils import (
        basic_gl_setup,
        adjust_gl_view,
        clear_gl_screen,
        make_coord_system_pixel_based,
        make_coord_system_norm_based,
    )

    # check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version

    assert pyglui_version >= "0.7"

    # monitoring
    import psutil

    # helpers/utils
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t
    from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
    from version_utils import VersionFormat

    # Plug-ins
    from plugin import Plugin_List, import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins
    from recorder import Recorder
    from show_calibration import Show_Calibration
    from display_recent_gaze import Display_Recent_Gaze
    from pupil_server import Pupil_Server
    from pupil_sync import Pupil_Sync
    from marker_detector import Marker_Detector
    from log_display import Log_Display
    from annotations import Annotation_Capture

    # create logger for the context of this function

    # UI Platform tweaks
    if platform.system() == "Linux":
        scroll_factor = 10.0
        window_position_default = (0, 0)
    elif platform.system() == "Windows":
        scroll_factor = 1.0
        window_position_default = (8, 31)
    else:
        scroll_factor = 1.0
        window_position_default = (0, 0)

    # g_pool holds variables for this process
    g_pool = Global_Container()

    # make some constants avaiable
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.app = "capture"
    g_pool.pupil_queue = pupil_queue
    g_pool.timebase = timebase
    # g_pool.lauchner_pipe = lauchner_pipe
    g_pool.eye_pipes = eye_pipes
    g_pool.eyes_are_alive = eyes_are_alive

    # manage plugins
    runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, "plugins"))
    user_launchable_plugins = [
        Show_Calibration,
        Pupil_Server,
        Pupil_Sync,
        Marker_Detector,
        Annotation_Capture,
    ] + runtime_plugins
    system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
    plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index, plugin_by_index))
    default_plugins = [
        ("Log_Display", {}),
        ("Dummy_Gaze_Mapper", {}),
        ("Display_Recent_Gaze", {}),
        ("Screen_Marker_Calibration", {}),
        ("Recorder", {}),
    ]

    # Callback functions
    def on_resize(window, w, h):
        if not g_pool.iconified:
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfw.glfwGetCursorPos(window)
        pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
        pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    tick = delta_t()

    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, "user_settings_world"))
    if session_settings.get("version", VersionFormat("0.0")) < g_pool.version:
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {"frame_size": (1280, 720), "frame_rate": 30}
    previous_settings = session_settings.get("capture_settings", None)
    if previous_settings and previous_settings["name"] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        lauchner_pipe.send("Exit")
        return

    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get("pupil_confidence_threshold", 0.6)
    g_pool.detection_mapping_mode = session_settings.get("detection_mapping_mode", "2d")
    g_pool.active_calibration_plugin = None

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def launch_eye_process(eye_id, blocking=False):
        if eyes_are_alive[eye_id].value:
            logger.error("Eye%s process already running." % eye_id)
            return
        lauchner_pipe.send(eye_id)
        eye_pipes[eye_id].send(("Set_Detection_Mapping_Mode", g_pool.detection_mapping_mode))

        if blocking:
            # wait for ready message from eye to sequentialize startup
            eye_pipes[eye_id].send("Ping")
            eye_pipes[eye_id].recv()

        logger.warning("Eye %s process started." % eye_id)

    def stop_eye_process(eye_id, blocking=False):
        if eyes_are_alive[eye_id].value:
            eye_pipes[eye_id].send("Exit")
            if blocking:
                raise NotImplementedError()

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        if new_mode == "2d":
            for p in g_pool.plugins:
                if "Vector_Gaze_Mapper" in p.class_name:
                    logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
                    p.alive = False
            g_pool.plugins.clean()
        for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes):
            if alive.value:
                pipe.send(("Set_Detection_Mapping_Mode", new_mode))
        g_pool.detection_mapping_mode = new_mode

    # window and gl setup
    glfw.glfwInit()
    width, height = session_settings.get("window_size", (frame.width, frame.height))
    main_window = glfw.glfwCreateWindow(width, height, "World")
    window_pos = session_settings.get("window_position", window_position_default)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window

    # setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get("gui_scale", 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos="left")
    general_settings = ui.Growing_Menu("General")
    general_settings.append(
        ui.Slider("scale", g_pool.gui, setter=set_scale, step=0.05, min=1.0, max=2.5, label="Interface size")
    )
    general_settings.append(
        ui.Button("Reset window size", lambda: glfw.glfwSetWindowSize(main_window, frame.width, frame.height))
    )
    general_settings.append(
        ui.Selector(
            "detection_mapping_mode",
            g_pool,
            label="detection & mapping mode",
            setter=set_detection_mapping_mode,
            selection=["2d", "3d"],
        )
    )
    general_settings.append(
        ui.Switch(
            "eye0_process",
            label="detect eye 0",
            setter=lambda alive: start_stop_eye(0, alive),
            getter=lambda: eyes_are_alive[0].value,
        )
    )
    general_settings.append(
        ui.Switch(
            "eye1_process",
            label="detect eye 1",
            setter=lambda alive: start_stop_eye(1, alive),
            getter=lambda: eyes_are_alive[1].value,
        )
    )
    general_settings.append(
        ui.Selector(
            "Open plugin",
            selection=user_launchable_plugins,
            labels=[p.__name__.replace("_", " ") for p in user_launchable_plugins],
            setter=open_plugin,
            getter=lambda: "Select to load",
        )
    )
    general_settings.append(
        ui.Slider("pupil_confidence_threshold", g_pool, step=0.01, min=0.0, max=1.0, label="Minimum pupil confidence")
    )
    general_settings.append(ui.Info_Text("Capture Version: %s" % g_pool.version))
    g_pool.sidebar.append(general_settings)

    g_pool.calibration_menu = ui.Growing_Menu("Calibration")
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.capture.init_gui(g_pool.sidebar)

    # plugins that are loaded based on user settings from previous session
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get("loaded_plugins", default_plugins))

    # We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(
        0,
        ui.Selector(
            "active_calibration_plugin",
            getter=lambda: g_pool.active_calibration_plugin.__class__,
            selection=calibration_plugins,
            labels=[p.__name__.replace("_", " ") for p in calibration_plugins],
            setter=open_plugin,
            label="Method",
        ),
    )

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfw.glfwSetKeyCallback(main_window, on_key)
    glfw.glfwSetCharCallback(main_window, on_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    # refresh speed settings
    glfw.glfwSwapInterval(0)

    # trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    # now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get("ui_config", {})

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = "CPU %0.1f"

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    if session_settings.get("eye1_process_alive", False):
        launch_eye_process(1, blocking=True)
    if session_settings.get("eye0_process_alive", True):
        launch_eye_process(0, blocking=False)

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        # update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1.0 / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        # a dictionary that allows plugins to post and read events
        events = {}

        # report time between now and the last loop interation
        events["dt"] = get_dt()

        # receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p["confidence"])
        events["pupil_positions"] = recent_pupil_positions

        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n["_notify_time_"] < time():
                del n["_notify_time_"]
                del g_pool.delayed_notifications[n["subject"]]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifications:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        if g_pool.iconified:
            pass
        else:
            g_pool.image_tex.update_from_frame(frame)

        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height, frame.width, 3))
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        if not g_pool.iconified:
            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
    session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
    session_settings["pupil_confidence_threshold"] = g_pool.pupil_confidence_threshold
    session_settings["gui_scale"] = g_pool.gui.scale
    session_settings["ui_config"] = g_pool.gui.configuration
    session_settings["capture_settings"] = g_pool.capture.settings
    session_settings["window_size"] = glfw.glfwGetWindowSize(main_window)
    session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
    session_settings["version"] = g_pool.version
    session_settings["eye0_process_alive"] = eyes_are_alive[0].value
    session_settings["eye1_process_alive"] = eyes_are_alive[1].value
    session_settings["detection_mapping_mode"] = g_pool.detection_mapping_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()
    cap.close()

    # shut down eye processes:
    stop_eye_process(0)
    stop_eye_process(1)

    # shut down laucher
    lauchner_pipe.send("Exit")

    logger.debug("world process done")
Beispiel #34
0
def world(pupil_queue, timebase, lauchner_pipe, eye_pipes, eyes_are_alive,
          user_dir, version, cap_src):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    import logging
    # Set up root logger for this process before doing imports of logged modules.
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(user_dir, 'world.log'), mode='w')
    fh.setLevel(logger.level)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level + 10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        'World Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter(
        'WORLD Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    #silence noisy modules
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logging.getLogger("libav").setLevel(logging.ERROR)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmfull but unnessasary.

    #general imports
    from time import time
    import numpy as np

    #display
    import glfw
    from pyglui import ui, graph, cygl
    from pyglui.cygl.utils import Named_Texture
    from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based

    #check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version
    assert pyglui_version >= '0.8'

    #monitoring
    import psutil

    # helpers/utils
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t
    from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
    from version_utils import VersionFormat
    import audio

    # Plug-ins
    from plugin import Plugin_List, import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins
    from recorder import Recorder
    from show_calibration import Show_Calibration
    from display_recent_gaze import Display_Recent_Gaze
    from pupil_server import Pupil_Server
    from pupil_sync import Pupil_Sync
    from marker_detector import Marker_Detector
    from log_display import Log_Display
    from annotations import Annotation_Capture
    # create logger for the context of this function

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (0, 0)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (8, 31)
    else:
        scroll_factor = 1.0
        window_position_default = (0, 0)

    #g_pool holds variables for this process
    g_pool = Global_Container()

    # make some constants avaiable
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.app = 'capture'
    g_pool.pupil_queue = pupil_queue
    g_pool.timebase = timebase
    # g_pool.lauchner_pipe = lauchner_pipe
    g_pool.eye_pipes = eye_pipes
    g_pool.eyes_are_alive = eyes_are_alive

    #manage plugins
    runtime_plugins = import_runtime_plugins(
        os.path.join(g_pool.user_dir, 'plugins'))
    user_launchable_plugins = [
        Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector,
        Annotation_Capture
    ] + runtime_plugins
    system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
    plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index, plugin_by_index))
    default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
                       ('Display_Recent_Gaze', {}),
                       ('Screen_Marker_Calibration', {}), ('Recorder', {})]

    # Callback functions
    def on_resize(window, w, h):
        if not g_pool.iconified:
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfw.glfwGetCursorPos(window)
        pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
        pos = denormalize(
            pos,
            (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfw.glfwGetFramebufferSize(window)[0] /
            glfw.glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    tick = delta_t()

    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_world'))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()

    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size': (1280, 720), 'frame_rate': 30}
    previous_settings = session_settings.get('capture_settings', None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        lauchner_pipe.send("Exit")
        return

    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get(
        'pupil_confidence_threshold', .6)
    g_pool.detection_mapping_mode = session_settings.get(
        'detection_mapping_mode', '2d')
    g_pool.active_calibration_plugin = None

    audio.audio_mode = session_settings.get('audio_mode',
                                            audio.default_audio_mode)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def launch_eye_process(eye_id, blocking=False):
        if eyes_are_alive[eye_id].value:
            logger.error("Eye%s process already running." % eye_id)
            return
        lauchner_pipe.send(eye_id)
        eye_pipes[eye_id].send(
            ('Set_Detection_Mapping_Mode', g_pool.detection_mapping_mode))

        if blocking:
            #wait for ready message from eye to sequentialize startup
            eye_pipes[eye_id].send('Ping')
            eye_pipes[eye_id].recv()

        logger.warning('Eye %s process started.' % eye_id)

    def stop_eye_process(eye_id, blocking=False):
        if eyes_are_alive[eye_id].value:
            eye_pipes[eye_id].send('Exit')
            if blocking:
                raise NotImplementedError()

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        if new_mode == '2d':
            for p in g_pool.plugins:
                if "Vector_Gaze_Mapper" in p.class_name:
                    logger.warning(
                        "The gaze mapper is not supported in 2d mode. Please recalibrate."
                    )
                    p.alive = False
            g_pool.plugins.clean()
        for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes):
            if alive.value:
                pipe.send(('Set_Detection_Mapping_Mode', new_mode))
        g_pool.detection_mapping_mode = new_mode

    #window and gl setup
    glfw.glfwInit()
    width, height = session_settings.get('window_size',
                                         (frame.width, frame.height))
    main_window = glfw.glfwCreateWindow(width, height, "World")
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-350, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface size'))
    general_settings.append(
        ui.Button(
            'Reset window size', lambda: glfw.glfwSetWindowSize(
                main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('audio_mode', audio, selection=audio.audio_modes))
    general_settings.append(
        ui.Selector('detection_mapping_mode',
                    g_pool,
                    label='detection & mapping mode',
                    setter=set_detection_mapping_mode,
                    selection=['2d', '3d']))
    general_settings.append(
        ui.Switch('eye0_process',
                  label='Detect eye 0',
                  setter=lambda alive: start_stop_eye(0, alive),
                  getter=lambda: eyes_are_alive[0].value))
    general_settings.append(
        ui.Switch('eye1_process',
                  label='Detect eye 1',
                  setter=lambda alive: start_stop_eye(1, alive),
                  getter=lambda: eyes_are_alive[1].value))
    general_settings.append(
        ui.Selector('Open plugin',
                    selection=user_launchable_plugins,
                    labels=[
                        p.__name__.replace('_', ' ')
                        for p in user_launchable_plugins
                    ],
                    setter=open_plugin,
                    getter=lambda: "Select to load"))
    general_settings.append(
        ui.Slider('pupil_confidence_threshold',
                  g_pool,
                  step=.01,
                  min=0.,
                  max=1.,
                  label='Minimum pupil confidence'))
    general_settings.append(
        ui.Info_Text('Capture Version: %s' % g_pool.version))
    g_pool.sidebar.append(general_settings)

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.capture.init_gui(g_pool.sidebar)

    #plugins that are loaded based on user settings from previous session
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(
        g_pool, plugin_by_name,
        session_settings.get('loaded_plugins', default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(
        0,
        ui.Selector(
            'active_calibration_plugin',
            getter=lambda: g_pool.active_calibration_plugin.__class__,
            selection=calibration_plugins,
            labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
            setter=open_plugin,
            label='Method'))

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfw.glfwSetKeyCallback(main_window, on_key)
    glfw.glfwSetCharCallback(main_window, on_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    # refresh speed settings
    glfw.glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    if session_settings.get('eye1_process_alive', False):
        launch_eye_process(1, blocking=True)
    if session_settings.get('eye0_process_alive', True):
        launch_eye_process(0, blocking=False)

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        #a dictionary that allows plugins to post and read events
        events = {}

        #report time between now and the last loop interation
        events['dt'] = get_dt()

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p['confidence'])
        events['pupil_positions'] = recent_pupil_positions

        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifications:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        if g_pool.iconified:
            pass
        else:
            g_pool.image_tex.update_from_frame(frame)

        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height, frame.width, 3))
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        if not g_pool.iconified:
            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings[
        'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings['eye0_process_alive'] = eyes_are_alive[0].value
    session_settings['eye1_process_alive'] = eyes_are_alive[1].value
    session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
    session_settings['audio_mode'] = audio.audio_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()
    cap.close()

    #shut down eye processes:
    stop_eye_process(0)
    stop_eye_process(1)

    #shut down laucher
    lauchner_pipe.send("Exit")

    logger.debug("world process done")
Beispiel #35
0
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir,
                                          'eye%s.log' % eye_id),
                             mode='w')
    # fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level + 10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        'Eye' + str(eye_id) +
        ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter(
        'EYE' + str(eye_id) +
        ' Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (600, 300 * eye_id)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (600, 31 + 300 * eye_id)
    else:
        scroll_factor = 1.0
        window_position_default = (600, 300 * eye_id)

    # Callback functions
    def on_resize(window, w, h):
        if not g_pool.iconified:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(window)
            g_pool.gui.update_window(w, h)
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            glfwMakeContextCurrent(active_window)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_button(window, button, action, mods):
        if g_pool.display_mode == 'roi':
            if action == GLFW_RELEASE and u_r.active_edit_pt:
                u_r.active_edit_pt = False
                return  # if the roi interacts we dont what the gui to interact as well
            elif action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos, glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(
                    pos, (frame.width, frame.height))  # Position in img pixels
                if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
                                          u_r.handle_size + 40):
                    return  # if the roi interacts we dont what the gui to interact as well

        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

        if u_r.active_edit_pt:
            pos = normalize((x, y), glfwGetWindowSize(main_window))
            if g_pool.flip:
                pos = 1 - pos[0], 1 - pos[1]
            pos = denormalize(pos, (frame.width, frame.height))
            u_r.move_vertex(u_r.active_pt_idx, pos)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()
    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size': cap_size, 'frame_rate': 30}
    previous_settings = session_settings.get('capture_settings', None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    #signal world that we are ready to go
    pipe_to_world.send('eye%s process ready' % eye_id)

    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.flip = session_settings.get('flip', False)
    g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
    g_pool.display_mode_info_text = {
        'camera_image':
        "Raw eye camera image. This uses the least amount of CPU power",
        'roi':
        "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
        'algorithm':
        "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
    }
    # g_pool.draw_pupil = session_settings.get('draw_pupil',True)

    u_r = UIRoi(frame.img.shape)
    u_r.set(session_settings.get('roi', u_r.get()))

    writer = None

    pupil_detector = Canny_Detector(g_pool)

    # UI callback functions
    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def set_display_mode_info(val):
        g_pool.display_mode = val
        g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

    # Initialize glfw
    glfwInit()
    if g_pool.binocular:
        title = "Binocular eye %s" % eye_id
    else:
        title = 'Eye'
    width, height = session_settings.get('window_size',
                                         (frame.width, frame.height))
    main_window = glfwCreateWindow(width, height, title, None, None)
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl_init()

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    glfwSwapInterval(0)

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-300, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface Size'))
    general_settings.append(
        ui.Button(
            'Reset window size',
            lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('display_mode',
                    g_pool,
                    setter=set_display_mode_info,
                    selection=['camera_image', 'roi', 'algorithm'],
                    labels=['Camera Image', 'ROI', 'Algorithm'],
                    label="Mode"))
    general_settings.append(
        ui.Switch('flip', g_pool, label='Flip image display'))
    g_pool.display_mode_info = ui.Info_Text(
        g_pool.display_mode_info_text[g_pool.display_mode])
    general_settings.append(g_pool.display_mode_info)
    g_pool.sidebar.append(general_settings)
    g_pool.gui.append(g_pool.sidebar)
    # let the camera add its GUI
    g_pool.capture.init_gui(g_pool.sidebar)
    # let detector add its GUI
    pupil_detector.init_gui(g_pool.sidebar)

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetWindowCloseCallback(main_window, on_close)
    glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)

    #set the last saved window size
    on_resize(main_window, *glfwGetWindowSize(main_window))

    # load last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performance graphs
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    #create a timer to control window update frequency
    window_update_timer = timer(1 / 60.)

    def window_should_update():
        return next(window_update_timer)

    # Event loop
    while not g_pool.quit.value:
        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if pipe_to_world.poll():
            command, raw_mode = pipe_to_world.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %s" % record_path)
                timestamps_path = os.path.join(record_path,
                                               "eye%s_timestamps.npy" % eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = JPEG_Writer(video_path, cap.frame_rate)
                else:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = AV_Writer(video_path, cap.frame_rate)
                timestamps = []
            else:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write_video_frame(frame)
            timestamps.append(frame.timestamp)

        # pupil ellipse detection
        result = pupil_detector.detect(
            frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
        result['id'] = eye_id
        # stream the result
        g_pool.pupil_queue.put(result)

        # GL drawing
        if window_should_update():
            if not g_pool.iconified:
                glfwMakeContextCurrent(main_window)
                clear_gl_screen()

                # switch to work in normalized coordinate space
                if g_pool.display_mode == 'algorithm':
                    g_pool.image_tex.update_from_ndarray(frame.img)
                elif g_pool.display_mode in ('camera_image', 'roi'):
                    g_pool.image_tex.update_from_ndarray(frame.gray)
                else:
                    pass

                make_coord_system_norm_based(g_pool.flip)
                g_pool.image_tex.draw()
                # switch to work in pixel space
                make_coord_system_pixel_based((frame.height, frame.width, 3),
                                              g_pool.flip)

                if result['confidence'] > 0:
                    if result.has_key('axes'):
                        pts = cv2.ellipse2Poly((int(
                            result['center'][0]), int(result['center'][1])),
                                               (int(result['axes'][0] / 2),
                                                int(result['axes'][1] / 2)),
                                               int(result['angle']), 0, 360,
                                               15)
                        cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
                    cygl_draw_points([result['center']],
                                     size=20,
                                     color=cygl_rgba(1., 0., 0., .5),
                                     sharpness=1.)

                # render graphs
                graph.push_view()
                fps_graph.draw()
                cpu_graph.draw()
                graph.pop_view()

                # render GUI
                g_pool.gui.update()

                #render the ROI
                if g_pool.display_mode == 'roi':
                    u_r.draw(g_pool.gui.scale)

                #update screen
                glfwSwapBuffers(main_window)
            glfwPollEvents()

    # END while running

    # in case eye recording was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path, np.asarray(timestamps))

    glfwRestoreWindow(main_window)  #need to do this for windows os
    # save session persistent settings
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['roi'] = u_r.get()
    session_settings['flip'] = g_pool.flip
    session_settings['display_mode'] = g_pool.display_mode
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    pupil_detector.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    #flushing queue in case world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")
Beispiel #36
0
    def open(
        self,
        gui_monitor: GUIMonitor,
        title: str,
        is_fullscreen: bool = False,
        size: T.Tuple[int, int] = None,
        position: T.Tuple[int, int] = None,
    ):
        if self.is_open:
            # TODO: Warn that the window is already open
            return

        if not gui_monitor.is_available:
            raise ValueError(f"Window requires an available monitor.")

        has_fixed_size = (
            (size is not None) and (len(size) == 2) and (size[0] > 0) and (size[1] > 0)
        )
        if is_fullscreen and has_fixed_size:
            raise ValueError(
                f"Fullscreen is mutually exclusive to having a fixed size."
            )

        if position is None:
            if platform.system() == "Windows":
                position = (8, 90)
            else:
                position = (0, 0)

        if is_fullscreen:
            size = gui_monitor.size

        # NOTE: Always creating windowed window here, even if in fullscreen mode. On
        # windows you might experience a black screen for up to 1 sec when creating
        # a blank window directly in fullscreen mode. By creating it windowed and
        # then switching to fullscreen it will stay white the entire time.
        self.__gl_handle = glfw.glfwCreateWindow(
            *size, title, share=glfw.glfwGetCurrentContext()
        )

        if not is_fullscreen:
            glfw.glfwSetWindowPos(self.__gl_handle, *position)

        # Register callbacks
        glfw.glfwSetFramebufferSizeCallback(self.__gl_handle, self.on_resize)
        glfw.glfwSetKeyCallback(self.__gl_handle, self.on_key)
        glfw.glfwSetMouseButtonCallback(self.__gl_handle, self.on_mouse_button)
        self.on_resize(self.__gl_handle, *glfw.glfwGetFramebufferSize(self.__gl_handle))

        # gl_state settings
        with self._switch_to_current_context():
            basic_gl_setup()
            glfw.glfwSwapInterval(0)

        if is_fullscreen:
            # Switch to full screen here. See NOTE above at glfwCreateWindow().
            glfw.glfwSetWindowMonitor(
                self.__gl_handle,
                gui_monitor.unsafe_handle,
                0,
                0,
                *gui_monitor.size,
                gui_monitor.refresh_rate,
            )
Beispiel #37
0
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
                app_version, debug):
    # general imports
    import logging

    # networking
    import zmq
    import zmq_tools
    from time import sleep

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:
        import glfw
        import gl_utils
        from OpenGL.GL import glClearColor
        from version_utils import VersionFormat
        from file_methods import Persistent_Dict
        from pyglui.pyfontstash import fontstash
        from pyglui.ui import get_roboto_font_path
        import player_methods as pm
        from pupil_recording import (
            assert_valid_recording_type,
            InvalidRecordingException,
        )
        from pupil_recording.update import update_recording

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        def on_drop(window, count, paths):
            nonlocal rec_dir
            rec_dir = paths[0].decode("utf-8")

        if rec_dir:
            try:
                assert_valid_recording_type(rec_dir)
            except InvalidRecordingException as err:
                logger.error(str(err))
                rec_dir = None
        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != app_version:
            logger.info(
                "Session setting are from a  different version of this app. I will not use those."
            )
            session_settings.clear()
        w, h = session_settings.get("window_size", (1280, 720))
        window_pos = session_settings.get("window_position",
                                          window_position_default)

        glfw.glfwInit()
        glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0)
        window = glfw.glfwCreateWindow(w, h, "Pupil Player")
        glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1)

        glfw.glfwMakeContextCurrent(window)
        glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1])
        glfw.glfwSetDropCallback(window, on_drop)

        glfont = fontstash.Context()
        glfont.add_font("roboto", get_roboto_font_path())
        glfont.set_align_string(v_align="center", h_align="middle")
        glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
        gl_utils.basic_gl_setup()
        glClearColor(0.5, 0.5, 0.5, 0.0)
        text = "Drop a recording directory onto this window."
        tip = "(Tip: You can drop a recording directory onto the app icon.)"

        # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."

        def display_string(string, font_size, center_y):
            x = w / 2 * hdpi_factor
            y = center_y * hdpi_factor

            glfont.set_size(font_size * hdpi_factor)

            glfont.set_blur(10.5)
            glfont.set_color_float((0.0, 0.0, 0.0, 1.0))
            glfont.draw_text(x, y, string)

            glfont.set_blur(0.96)
            glfont.set_color_float((1.0, 1.0, 1.0, 1.0))
            glfont.draw_text(x, y, string)

        while not glfw.glfwWindowShouldClose(
                window) and not process_was_interrupted:

            fb_size = glfw.glfwGetFramebufferSize(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            gl_utils.adjust_gl_view(*fb_size)

            if rec_dir:
                try:
                    assert_valid_recording_type(rec_dir)
                    logger.info(
                        "Starting new session with '{}'".format(rec_dir))
                    text = "Updating recording format."
                    tip = "This may take a while!"
                except InvalidRecordingException as err:
                    logger.error(str(err))
                    if err.recovery:
                        text = err.reason
                        tip = err.recovery
                    else:
                        text = "Invalid recording"
                        tip = err.reason
                    rec_dir = None

            gl_utils.clear_gl_screen()

            display_string(text, font_size=51, center_y=216)
            for idx, line in enumerate(tip.split("\n")):
                tip_font_size = 42
                center_y = 288 + tip_font_size * idx * 1.2
                display_string(line,
                               font_size=tip_font_size,
                               center_y=center_y)

            glfw.glfwSwapBuffers(window)

            if rec_dir:
                try:
                    update_recording(rec_dir)
                except AssertionError as err:
                    logger.error(str(err))
                    tip = "Oops! There was an error updating the recording."
                    rec_dir = None
                except InvalidRecordingException as err:
                    logger.error(str(err))
                    if err.recovery:
                        text = err.reason
                        tip = err.recovery
                    else:
                        text = "Invalid recording"
                        tip = err.reason
                    rec_dir = None
                else:
                    glfw.glfwSetWindowShouldClose(window, True)

            glfw.glfwPollEvents()

        session_settings["window_position"] = glfw.glfwGetWindowPos(window)
        session_settings.close()
        glfw.glfwDestroyWindow(window)
        if rec_dir:
            ipc_pub.notify({
                "subject": "player_process.should_start",
                "rec_dir": rec_dir
            })

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error(
            "Process player_drop crashed with trace:\n{}".format(trace))

    finally:
        sleep(1.0)
Beispiel #38
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version):
    # general imports
    from time import sleep
    import logging
    import errno
    from glob import glob
    from time import time
    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        # imports
        from file_methods import Persistent_Dict, load_object

        # display
        import glfw
        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils
        # capture
        from video_capture import File_Source, EndofVideoFileError

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation
        from vis_scan_path import Vis_Scan_Path
        from vis_eye_video_overlay import Vis_Eye_Video_Overlay
        from seek_control import Seek_Control
        from video_export_launcher import Video_Export_Launcher
        from offline_surface_tracker import Offline_Surface_Tracker
        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from batch_exporter import Batch_Exporter, Batch_Export
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producers import Gaze_From_Recording, Offline_Calibration
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection

        assert VersionFormat(pyglui_version) >= VersionFormat(
            '1.17'), 'pyglui out of date, please upgrade to newest version'

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, 'plugins'))
        system_plugins = [
            Log_Display, Seek_Control, Plugin_Manager, System_Graphs,
            Batch_Export, System_Timelines
        ]
        user_plugins = [
            Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points,
            Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path,
            Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter,
            Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter,
            Annotation_Player, Log_History, Pupil_From_Recording,
            Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor

            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            for x in range(count):
                new_rec_dir = paths[x].decode('utf-8')
                if is_pupil_rec_dir(new_rec_dir):
                    logger.debug(
                        "Starting new session with '{}'".format(new_rec_dir))
                    ipc_pub.notify({
                        "subject": "player_drop_process.should_start",
                        "rec_dir": new_rec_dir
                    })
                    glfw.glfwSetWindowShouldClose(window, True)
                else:
                    logger.error("'{}' is not a valid pupil recording".format(
                        new_rec_dir))

        tick = delta_t()

        def get_dt():
            return next(tick)

        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264',
                                          '.mjpeg')
        ][0]
        pupil_data_path = os.path.join(rec_dir, "pupil_data")

        meta_info = load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info('Application Version: {}'.format(app_version))
        logger.info('System Info: {}'.format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = Global_Container()
        g_pool.app = 'player'
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        # sets itself to g_pool.capture
        File_Source(g_pool, video_path)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.capture.playback_speed = session_settings.get(
            'playback_speed', 1.)

        width, height = session_settings.get('window_size',
                                             g_pool.capture.frame_size)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(
            width, height, "Pupil Player: " + meta_info["Recording Name"] +
            " - " + rec_dir.split(os.path.sep)[-1], None, None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1])
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.pupil_data = load_object(pupil_data_path)
        g_pool.binocular = meta_info.get('Eye Mode',
                                         'monocular') == 'binocular'
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.
        g_pool.new_seek = True
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            'min_data_confidence', 0.6)

        g_pool.pupil_positions = []
        g_pool.gaze_positions = []
        g_pool.fixations = []

        g_pool.notifications_by_frame = correlate_data(
            g_pool.pupil_data['notifications'], g_pool.timestamps)
        g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps
                                           ]  # populated by producer`
        g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps
                                          ]  # populated by producer
        g_pool.fixations_by_frame = [
            [] for x in g_pool.timestamps
        ]  # populated by the fixation detector plugin

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {'subject': 'min_data_confidence_changed'}
            notification['_notify_time_'] = time() + .8
            g_pool.ipc_pub.notify(notification)

        def open_plugin(plugin):
            if plugin == "Select to load":
                return
            g_pool.plugins.add(plugin)

        def purge_plugins():
            for p in g_pool.plugins:
                if p.__class__ in user_plugins:
                    p.alive = False
            g_pool.plugins.clean()

        def do_export(_):
            export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right
            export_dir = os.path.join(g_pool.rec_dir, 'exports',
                                      '{}-{}'.format(*export_range))
            try:
                os.makedirs(export_dir)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    logger.error("Could not create export dir")
                    raise e
                else:
                    overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting."
                    logger.warning(overwrite_warning.format(*export_range))
            else:
                logger.info('Created export dir at "{}"'.format(export_dir))

            notification = {
                'subject': 'should_export',
                'range': export_range,
                'export_dir': export_dir
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({
                'subject': 'player_process.should_start',
                'rec_dir': rec_dir,
                'delay': 2.
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos='hidden')
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu('User Timelines',
                                                 pos=(0., -150.),
                                                 size=(0., 0.),
                                                 header_pos='headline')
        g_pool.user_timelines.color = RGBA(a=0.)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, g_pool.capture.frame_size[0], g_pool.capture.
                    frame_size[1])))
        general_settings.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.8, .9, 1., 1.1, 1.2] +
                        list(np.arange(1.5, 5.1, .5)),
                        label='Interface Size'))
        general_settings.append(
            ui.Info_Text('Player Version: {}'.format(g_pool.version)))
        general_settings.append(
            ui.Info_Text('Capture Version: {}'.format(
                meta_info['Capture Software Version'])))
        general_settings.append(
            ui.Info_Text('Data Format Version: {}'.format(
                meta_info['Data Format Version'])))
        general_settings.append(
            ui.Slider('min_data_confidence',
                      g_pool,
                      setter=set_data_confidence,
                      step=.05,
                      min=0.0,
                      max=1.0,
                      label='Confidence threshold'))
        general_settings.append(
            ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed',
                       general_settings,
                       label=chr(0xe8b8),
                       on_val=False,
                       off_val=True,
                       setter=toggle_general_settings,
                       label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb('export',
                                        label=chr(0xe2c5),
                                        getter=lambda: False,
                                        setter=do_export,
                                        hotkey='e',
                                        label_font='pupil_icons')
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}),
                           ('Log_Display', {}), ('Raw_Data_Exporter', {}),
                           ('Vis_Polyline', {}), ('Vis_Circle', {}),
                           ('System_Graphs', {}), ('System_Timelines', {}),
                           ('Video_Export_Launcher', {}),
                           ('Pupil_From_Recording', {}),
                           ('Gaze_From_Recording', {})]
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'start_plugin':
                g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
                                   args=n.get('args', {}))
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({
                    'subject': 'meta.doc',
                    'actor': g_pool.app,
                    'doc': player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            'subject': 'meta.doc',
                            'actor': p.class_name,
                            'doc': p.on_notify.__doc__
                        })

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # grab new frame
            if g_pool.capture.play or g_pool.new_seek:
                g_pool.new_seek = False
                try:
                    new_frame = g_pool.capture.get_frame()
                except EndofVideoFileError:
                    # end of video logic: pause at last frame.
                    g_pool.capture.play = False
                    logger.warning("end of video")

            frame = new_frame.copy()
            events = {}
            events['frame'] = frame
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events['pupil_positions'] = []
            events['gaze_positions'] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture._recent_frame = frame
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ''
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in user_input.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in user_input.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                glfw.glfwSwapBuffers(main_window)

            # present frames at appropriate speed
            g_pool.capture.wait(frame)
            glfw.glfwPollEvents()

        session_settings['playback_speed'] = g_pool.capture.playback_speed
        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['min_data_confidence'] = g_pool.min_data_confidence
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.capture.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Player crashed with trace:\n{}'.format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'player_process.stopped'})
        sleep(1.0)
Beispiel #39
0
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version,
        eye_id, cap_src):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """
    is_alive = Is_Alive_Manager(is_alive_flag)
    with is_alive:
        import logging
        # Set up root logger for this process before doing imports of logged modules.
        logger = logging.getLogger()
        logger.setLevel(logging.INFO)
        # remove inherited handlers
        logger.handlers = []
        # create file handler which logs even debug messages
        fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id),
                                 mode='w')
        # fh.setLevel(logging.DEBUG)
        # create console handler with a higher log level
        ch = logging.StreamHandler()
        ch.setLevel(logger.level + 10)
        # create formatter and add it to the handlers
        formatter = logging.Formatter(
            'Eye' + str(eye_id) +
            ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        formatter = logging.Formatter(
            'EYE' + str(eye_id) +
            ' Process [%(levelname)s] %(name)s : %(message)s')
        ch.setFormatter(formatter)
        # add the handlers to the logger
        logger.addHandler(fh)
        logger.addHandler(ch)
        #silence noisy modules
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logging.getLogger("libav").setLevel(logging.ERROR)
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        # We deferr the imports becasue of multiprocessing.
        # Otherwise the world process each process also loads the other imports.

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture
        from OpenGL.GL import GL_LINE_LOOP
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based
        from ui_roi import UIRoi
        #monitoring
        import psutil

        # helpers/utils
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
        from av_writer import JPEG_Writer, AV_Writer

        # Pupil detectors
        from pupil_detectors import Canny_Detector, Detector_2D, Detector_3D
        pupil_detectors = {
            Canny_Detector.__name__: Canny_Detector,
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D
        }

        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600, 31 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.pupil_queue = pupil_queue
        g_pool.timebase = timebase

        # Callback functions
        def on_resize(window, w, h):
            if not g_pool.iconified:
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w, h)
                graph.adjust_size(w, h)
                adjust_gl_view(w, h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_button(window, button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return  # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    pos = denormalize(
                        pos,
                        (frame.width, frame.height))  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                            pos, g_pool.u_r.handle_size + 40,
                            g_pool.u_r.handle_size + 40):
                        return  # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, (frame.width, frame.height))
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
        if session_settings.get("version",
                                VersionFormat('0.0')) < g_pool.version:
            logger.info(
                "Session setting are from older version of this app. I will not use those."
            )
            session_settings.clear()
        # Initialize capture
        cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
        default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
        previous_settings = session_settings.get('capture_settings', None)
        if previous_settings and previous_settings['name'] == cap.name:
            cap.settings = previous_settings
        else:
            cap.settings = default_settings

        # Test capture
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Could not retrieve image from capture")
            cap.close()
            return

        #signal world that we are ready to go
        # pipe_to_world.send('eye%s process ready'%eye_id)

        # any object we attach to the g_pool object *from now on* will only be visible to this process!
        # vars should be declared here to make them visible to the code reader.
        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.flip = session_settings.get('flip', False)
        g_pool.display_mode = session_settings.get('display_mode',
                                                   'camera_image')
        g_pool.display_mode_info_text = {
            'camera_image':
            "Raw eye camera image. This uses the least amount of CPU power",
            'roi':
            "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
            'algorithm':
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
        }

        g_pool.u_r = UIRoi(frame.img.shape)
        g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))

        def on_frame_size_change(new_size):
            g_pool.u_r = UIRoi((new_size[1], new_size[0]))

        cap.on_frame_size_change = on_frame_size_change

        writer = None

        pupil_detector_settings = session_settings.get(
            'pupil_detector_settings', None)
        last_pupil_detector = pupil_detectors[session_settings.get(
            'last_pupil_detector', Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,
                                                    pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)

        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s" % eye_id
        width, height = session_settings.get('window_size',
                                             (frame.width, frame.height))
        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_frame(frame)
        glfw.glfwSwapInterval(0)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale', 1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                           pos=(-300, 0),
                                           size=(0, 0),
                                           header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(
            ui.Slider('scale',
                      g_pool.gui,
                      setter=set_scale,
                      step=.05,
                      min=1.,
                      max=2.5,
                      label='Interface Size'))
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, frame.width, frame.height)))
        general_settings.append(
            ui.Switch('flip', g_pool, label='Flip image display'))
        general_settings.append(
            ui.Selector('display_mode',
                        g_pool,
                        setter=set_display_mode_info,
                        selection=['camera_image', 'roi', 'algorithm'],
                        labels=['Camera Image', 'ROI', 'Algorithm'],
                        label="Mode"))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.sidebar.append(general_settings)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector(
            'pupil_detector',
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Canny_Detector, Detector_2D, Detector_3D],
            labels=[
                'Python 2D detector', 'C++ 2d detector', 'C++ 3d detector'
            ],
            label="Detection method")
        general_settings.append(detector_selector)

        # let detector add its GUI
        g_pool.pupil_detector.init_gui(g_pool.sidebar)
        # let the camera add its GUI
        g_pool.capture.init_gui(g_pool.sidebar)

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_key)
        glfw.glfwSetCharCallback(main_window, on_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = frame.timestamp

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"

        #create a timer to control window update frequency
        window_update_timer = timer(1 / 60.)

        def window_should_update():
            return next(window_update_timer)

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if pipe_to_world.poll():
                cmd = pipe_to_world.recv()
                if cmd == 'Exit':
                    break
                elif cmd == "Ping":
                    pipe_to_world.send("Pong")
                    command = None
                else:
                    command, payload = cmd
                if command == 'Set_Detection_Mapping_Mode':
                    if payload == '3d':
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    else:
                        set_detector(Detector_2D)
                        detector_selector.read_only = False

            else:
                command = None

            # Get an image from the grabber
            try:
                frame = cap.get_frame()
            except CameraCaptureError:
                logger.error("Capture from Camera Failed. Stopping.")
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                cap.seek_to_frame(0)
                frame = cap.get_frame()

            #update performace graphs
            t = frame.timestamp
            dt, ts = t - ts, t
            try:
                fps_graph.add(1. / dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()

            ###  RECORDING of Eye Video (on demand) ###
            # Setup variables and lists for recording
            if 'Rec_Start' == command:
                record_path, raw_mode = payload
                logger.info("Will save eye video to: %s" % record_path)
                timestamps_path = os.path.join(record_path,
                                               "eye%s_timestamps.npy" % eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = JPEG_Writer(video_path, cap.frame_rate)
                else:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = AV_Writer(video_path, cap.frame_rate)
                timestamps = []
            elif 'Rec_Stop' == command:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)

            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(
                frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            g_pool.pupil_queue.put(result)

            # GL drawing
            if window_should_update():
                if not g_pool.iconified:
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image', 'roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()
                    # switch to work in pixel space
                    make_coord_system_pixel_based(
                        (frame.height, frame.width, 3), g_pool.flip)

                    if result['confidence'] > 0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly(
                                (int(result['ellipse']['center'][0]),
                                 int(result['ellipse']['center'][1])),
                                (int(result['ellipse']['axes'][0] / 2),
                                 int(result['ellipse']['axes'][1] / 2)),
                                int(result['ellipse']['angle']), 0, 360, 15)
                            draw_polyline(pts, 1, RGBA(1., 0, 0, .5))
                        draw_points([result['ellipse']['center']],
                                    size=20,
                                    color=RGBA(1., 0., 0., .5),
                                    sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize(
                )  #detector decides if we visualize or not

        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path, np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = g_pool.version
        session_settings[
            'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        cap.close()

        logger.debug("Process done")
Beispiel #40
0
def eye(g_pool,cap_src,cap_size):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye.log'),mode='w')
    fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logging.WARNING)
    # create formatter and add it to the handlers
    formatter = logging.Formatter('EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter('E Y E Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)


    # Callback functions
    def on_resize(window,w, h):
        adjust_gl_view(w,h,window)
        norm_size = normalize((w,h),glfwGetWindowSize(window))
        fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
        atb.TwWindowSize(*map(int,fb_size))


    def on_key(window, key, scancode, action, mods):
        if not atb.TwEventKeyboardGLFW(key,int(action == GLFW_PRESS)):
            if action == GLFW_PRESS:
                if key == GLFW_KEY_ESCAPE:
                    on_close(window)

    def on_char(window,char):
        if not atb.TwEventCharGLFW(char,1):
            pass

    def on_button(window,button, action, mods):
        if not atb.TwEventMouseButtonGLFW(button,int(action == GLFW_PRESS)):
            if action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos,glfwGetWindowSize(window))
                pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels
                u_r.setStart(pos)
                bar.draw_roi.value = 1
            else:
                bar.draw_roi.value = 0

    def on_pos(window,x, y):
        norm_pos = normalize((x,y),glfwGetWindowSize(window))
        fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window))
        if atb.TwMouseMotion(int(fb_x),int(fb_y)):
            pass

        if bar.draw_roi.value == 1:
            pos = denormalize(norm_pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels
            u_r.setEnd(pos)

    def on_scroll(window,x,y):
        if not atb.TwMouseWheel(int(x)):
            pass

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')


    # Helper functions called by the main atb bar
    def start_roi():
        bar.display.value = 1
        bar.draw_roi.value = 2

    def update_fps():
        old_time, bar.timestamp = bar.timestamp, time()
        dt = bar.timestamp - old_time
        if dt:
            bar.fps.value += .05 * (1. / dt - bar.fps.value)
            bar.dt.value = dt

    def get_from_data(data):
        """
        helper for atb getter and setter use
        """
        return data.value


    # load session persistent settings
    session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_eye'),protocol=2)
    def load(var_name,default):
        return session_settings.get(var_name,default)
    def save(var_name,var):
        session_settings[var_name] = var

    # Initialize capture
    cap = autoCreateCapture(cap_src, cap_size,timebase=g_pool.timebase)

    if cap is None:
        logger.error("Did not receive valid Capture")
        return
    # check if it works
    frame = cap.get_frame()
    if frame.img is None:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return
    height,width = frame.img.shape[:2]

    u_r = Roi(frame.img.shape)
    u_r.set(load('roi',default=None))

    writer = None

    pupil_detector = Canny_Detector(g_pool)

    atb.init()
    # Create main ATB Controls
    bar = atb.Bar(name = "Eye", label="Display",
            help="Scene controls", color=(50, 50, 50), alpha=100,
            text='light', position=(10, 10),refresh=.3, size=(200, 100))
    bar.fps = c_float(0.0)
    bar.timestamp = time()
    bar.dt = c_float(0.0)
    bar.sleep = c_float(0.0)
    bar.display = c_int(load('bar.display',0))
    bar.draw_pupil = c_bool(load('bar.draw_pupil',True))
    bar.draw_roi = c_int(0)

    dispay_mode_enum = atb.enum("Mode",{"Camera Image":0,
                                        "Region of Interest":1,
                                        "Algorithm":2,
                                        "CPU Save": 3})

    bar.add_var("FPS",bar.fps, step=1.,readonly=True)
    bar.add_var("Mode", bar.display,vtype=dispay_mode_enum, help="select the view-mode")
    bar.add_var("Show_Pupil_Point", bar.draw_pupil)
    bar.add_button("Draw_ROI", start_roi, help="drag on screen to select a region of interest")

    bar.add_var("SlowDown",bar.sleep, step=0.01,min=0.0)
    bar.add_var("SaveSettings&Exit", g_pool.quit)

    cap.create_atb_bar(pos=(220,10))

    # create a bar for the detector
    pupil_detector.create_atb_bar(pos=(10,120))


    glfwInit()
    window = glfwCreateWindow(width, height, "Eye", None, None)
    glfwMakeContextCurrent(window)

    # Register callbacks window
    glfwSetWindowSizeCallback(window,on_resize)
    glfwSetWindowCloseCallback(window,on_close)
    glfwSetKeyCallback(window,on_key)
    glfwSetCharCallback(window,on_char)
    glfwSetMouseButtonCallback(window,on_button)
    glfwSetCursorPosCallback(window,on_pos)
    glfwSetScrollCallback(window,on_scroll)

    glfwSetWindowPos(window,800,0)
    on_resize(window,width,height)

    # gl_state settings
    basic_gl_setup()

    # refresh speed settings
    glfwSwapInterval(0)


    # event loop
    while not g_pool.quit.value:
        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break

        update_fps()
        sleep(bar.sleep.value) # for debugging only


        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if g_pool.eye_rx.poll():
            command = g_pool.eye_rx.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %s"%record_path)
                video_path = os.path.join(record_path, "eye.avi")
                timestamps_path = os.path.join(record_path, "eye_timestamps.npy")
                writer = cv2.VideoWriter(video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (frame.img.shape[1], frame.img.shape[0]))
                timestamps = []
            else:
                logger.info("Done recording eye.")
                writer = None
                np.save(timestamps_path,np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write(frame.img)
            timestamps.append(frame.timestamp)


        # pupil ellipse detection
        result = pupil_detector.detect(frame,user_roi=u_r,visualize=bar.display.value == 2)
        # stream the result
        g_pool.pupil_queue.put(result)

        # VISUALIZATION direct visualizations on the frame.img data
        if bar.display.value == 1:
            # and a solid (white) frame around the user defined ROI
            r_img = frame.img[u_r.lY:u_r.uY,u_r.lX:u_r.uX]
            r_img[:,0] = 255,255,255
            r_img[:,-1]= 255,255,255
            r_img[0,:] = 255,255,255
            r_img[-1,:]= 255,255,255



        # GL-drawing
        clear_gl_screen()
        draw_gl_texture(frame.img,update=bar.display.value != 3)

        if result['norm_pupil'] is not None and bar.draw_pupil.value:
            if result.has_key('axes'):
                pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
                                        (int(result["axes"][0]/2),int(result["axes"][1]/2)),
                                        int(result["angle"]),0,360,15)
                draw_gl_polyline(pts,(1.,0,0,.5))
            draw_gl_point_norm(result['norm_pupil'],color=(1.,0.,0.,0.5))

        atb.draw()
        glfwSwapBuffers(window)
        glfwPollEvents()

    # END while running

    # in case eye reconding was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path,np.asarray(timestamps))


    # save session persistent settings
    save('roi',u_r.get())
    save('bar.display',bar.display.value)
    save('bar.draw_pupil',bar.draw_pupil.value)
    session_settings.close()

    pupil_detector.cleanup()
    cap.close()
    atb.terminate()
    glfwDestroyWindow(window)
    glfwTerminate()

    #flushing queue incase world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")
Beispiel #41
0
    def __init__(
        self,
        g_pool,
        window_size=window_size_default,
        window_position=window_position_default,
        gui_scale=1.0,
        ui_config={},
    ):
        super().__init__(g_pool)

        self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(*window_size, "Pupil Service")
        glfw.glfwSetWindowPos(main_window, *window_position)
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = gui_scale
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(0, 0), size=(0, 0), header_pos="headline"
        )
        g_pool.gui.append(g_pool.menubar)

        # Callback functions
        def on_resize(window, w, h):
            self.window_size = w, h
            self.hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * self.hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * self.hdpi_factor, y * self.hdpi_factor
            g_pool.gui.update_mouse(x, y)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *self.window_size)

        def set_window_size():
            glfw.glfwSetWindowSize(main_window, *window_size_default)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            self.notify_all({"subject": "clear_settings_process.should_start"})
            self.notify_all({"subject": "service_process.should_start", "delay": 2.0})

        g_pool.menubar.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.6, 0.8, 1.0, 1.2, 1.4],
                label="Interface size",
            )
        )

        g_pool.menubar.append(ui.Button("Reset window size", set_window_size))

        pupil_remote_addr = "{}:{}".format(
            socket.gethostbyname(socket.gethostname()), g_pool.preferred_remote_port
        )
        g_pool.menubar.append(
            ui.Text_Input(
                "pupil_remote_addr",
                getter=lambda: pupil_remote_addr,
                setter=lambda x: None,
                label="Pupil Remote address",
            )
        )

        g_pool.menubar.append(
            ui.Selector(
                "detection_mapping_mode",
                g_pool,
                label="Detection & mapping mode",
                setter=self.set_detection_mapping_mode,
                selection=["disabled", "2d", "3d"],
            )
        )
        g_pool.menubar.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: self.start_stop_eye(0, alive),
                getter=lambda: g_pool.eye_procs_alive[0].value,
            )
        )
        g_pool.menubar.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: self.start_stop_eye(1, alive),
                getter=lambda: g_pool.eye_procs_alive[1].value,
            )
        )

        g_pool.menubar.append(
            ui.Info_Text("Service Version: {}".format(g_pool.version))
        )

        g_pool.menubar.append(ui.Button("Restart with default settings", reset_restart))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        g_pool.gui.configuration = ui_config
        gl_utils.basic_gl_setup()

        on_resize(g_pool.main_window, *glfw.glfwGetFramebufferSize(main_window))
Beispiel #42
0
def world(pupil_queue,timebase,launcher_pipe,eye_pipes,eyes_are_alive,user_dir,version,cap_src):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    import logging
    # Set up root logger for this process before doing imports of logged modules.
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    #silence noisy modules
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    # create formatter
    formatter = logging.Formatter('%(processName)s - [%(levelname)s] %(name)s : %(message)s')
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(user_dir,'capture.log'),mode='w')
    fh.setLevel(logger.level)
    fh.setFormatter(formatter)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level+10)
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)


    #setup thread to recv log recrods from other processes.
    def log_loop(logging):
        import zmq
        ctx = zmq.Context()
        sub = ctx.socket(zmq.SUB)
        sub.bind('tcp://127.0.0.1:502020')
        sub.setsockopt(zmq.SUBSCRIBE, "")
        while True:
            record = sub.recv_pyobj()
            logger = logging.getLogger(record.name)
            logger.handle(record)

    import threading
    log_thread = threading.Thread(target=log_loop, args=(logging,))
    log_thread.setDaemon(True)
    log_thread.start()


    # create logger for the context of this function
    logger = logging.getLogger(__name__)


    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    #general imports
    from time import time,sleep
    import numpy as np

    #display
    import glfw
    from pyglui import ui,graph,cygl
    from pyglui.cygl.utils import Named_Texture
    from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen,make_coord_system_pixel_based,make_coord_system_norm_based

    #check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version
    assert pyglui_version >= '0.8'

    #monitoring
    import psutil

    # helpers/utils
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t, get_system_info
    from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
    from version_utils import VersionFormat
    import audio

    # Plug-ins
    from plugin import Plugin_List,import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins
    from recorder import Recorder
    from show_calibration import Show_Calibration
    from display_recent_gaze import Display_Recent_Gaze
    from pupil_server import Pupil_Server
    from pupil_sync import Pupil_Sync
    from surface_tracker import Surface_Tracker
    from log_display import Log_Display
    from annotations import Annotation_Capture
    from pupil_remote import Pupil_Remote
    from log_history import Log_History
    from game_controller import GameController

    logger.info('Application Version: %s'%version)
    logger.info('System Info: %s'%get_system_info())

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (0,0)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (8,31)
    else:
        scroll_factor = 1.0
        window_position_default = (0,0)



    #g_pool holds variables for this process
    g_pool = Global_Container()

    # make some constants avaiable
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.app = 'capture'
    g_pool.pupil_queue = pupil_queue
    g_pool.timebase = timebase
    # g_pool.launcher_pipe = launcher_pipe
    g_pool.eye_pipes = eye_pipes
    g_pool.eyes_are_alive = eyes_are_alive


    #manage plugins
    runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
    user_launchable_plugins = [GameController, Show_Calibration,Pupil_Remote,Pupil_Server,Pupil_Sync,Surface_Tracker,Annotation_Capture,Log_History]+runtime_plugins
    system_plugins  = [Log_Display,Display_Recent_Gaze,Recorder]
    plugin_by_index =  system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index,plugin_by_index))
    default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]


    # Callback functions
    def on_resize(window,w, h):
        if not g_pool.iconified:
            g_pool.gui.update_window(w,h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w,h)
            adjust_gl_view(w,h)
            for p in g_pool.plugins:
                p.on_window_resize(window,w,h)

    def on_iconify(window,iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)

    def on_button(window,button, action, mods):
        g_pool.gui.update_button(button,action,mods)
        pos = glfw.glfwGetCursorPos(window)
        pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
        pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos,button,action)

    def on_pos(window,x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
        x,y = x*hdpi_factor,y*hdpi_factor
        g_pool.gui.update_mouse(x,y)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*scroll_factor)


    tick = delta_t()
    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
    if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size':(1280,720),'frame_rate':30}
    previous_settings = session_settings.get('capture_settings',None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        launcher_pipe.send("Exit")
        return



    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
    g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','2d')
    g_pool.active_calibration_plugin = None


    audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode)

    def open_plugin(plugin):
        if plugin ==  "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def launch_eye_process(eye_id,blocking=False):
        if eyes_are_alive[eye_id].value:
            logger.error("Eye%s process already running."%eye_id)
            return
        launcher_pipe.send(eye_id)
        eye_pipes[eye_id].send( ('Set_Detection_Mapping_Mode',g_pool.detection_mapping_mode) )

        if blocking:
            #wait for ready message from eye to sequentialize startup
            eye_pipes[eye_id].send('Ping')
            eye_pipes[eye_id].recv()

        logger.warning('Eye %s process started.'%eye_id)

    def stop_eye_process(eye_id,blocking=False):
        if eyes_are_alive[eye_id].value:
            eye_pipes[eye_id].send('Exit')
            if blocking:
                while eyes_are_alive[eye_id].value:
                    sleep(.1)

    def start_stop_eye(eye_id,make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        if new_mode == '2d':
            for p in g_pool.plugins:
                if "Vector_Gaze_Mapper" in p.class_name:
                    logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
                    p.alive = False
            g_pool.plugins.clean()
        for alive, pipe in zip(g_pool.eyes_are_alive,g_pool.eye_pipes):
            if alive.value:
                pipe.send( ('Set_Detection_Mapping_Mode',new_mode) )
        g_pool.detection_mapping_mode = new_mode


    #window and gl setup
    glfw.glfwInit()
    width,height = session_settings.get('window_size',(frame.width, frame.height))
    main_window = glfw.glfwCreateWindow(width,height, "World")
    window_pos = session_settings.get('window_position',window_position_default)
    glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window



    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
    general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
    general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes))
    general_settings.append(ui.Selector('detection_mapping_mode',g_pool,label='detection & mapping mode',setter=set_detection_mapping_mode,selection=['2d','3d']))
    general_settings.append(ui.Switch('eye0_process',label='Detect eye 0',setter=lambda alive: start_stop_eye(0,alive),getter=lambda: eyes_are_alive[0].value ))
    general_settings.append(ui.Switch('eye1_process',label='Detect eye 1',setter=lambda alive: start_stop_eye(1,alive),getter=lambda: eyes_are_alive[1].value ))
    general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
                                        setter= open_plugin, getter=lambda: "Select to load"))
    general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
    general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
    g_pool.sidebar.append(general_settings)

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.capture.init_gui(g_pool.sidebar)

    #plugins that are loaded based on user settings from previous session
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
                                        setter= open_plugin,label='Method'))

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
    glfw.glfwSetKeyCallback(main_window,on_key)
    glfw.glfwSetCharCallback(main_window,on_char)
    glfw.glfwSetMouseButtonCallback(main_window,on_button)
    glfw.glfwSetCursorPosCallback(main_window,on_pos)
    glfw.glfwSetScrollCallback(main_window,on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    # refresh speed settings
    glfw.glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config',{})



    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260,130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"


    if session_settings.get('eye1_process_alive',False):
        launch_eye_process(1,blocking=True)
    if session_settings.get('eye0_process_alive',True):
        launch_eye_process(0,blocking=False)

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # Get an image from the grabber
        try:
            frame = g_pool.capture.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Starting Fake Capture.")
            settings = g_pool.capture.settings
            g_pool.capture.close()
            g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase)
            g_pool.capture.init_gui(g_pool.sidebar)
            g_pool.capture.settings = settings
            g_pool.notifications.append({'subject':'should_stop_recording'})
            continue
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt,ts = t-ts,t
        try:
            fps_graph.add(1./dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()


        #a dictionary that allows plugins to post and read events
        events = {}

        #report time between now and the last loop interation
        events['dt'] = get_dt()

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p['confidence'])
        events['pupil_positions'] = recent_pupil_positions


        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifications:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        if g_pool.iconified:
            pass
        else:
            g_pool.image_tex.update_from_frame(frame)

        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height,frame.width,3))
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        if not g_pool.iconified:
            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window) #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings['eye0_process_alive'] = eyes_are_alive[0].value
    session_settings['eye1_process_alive'] = eyes_are_alive[1].value
    session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
    session_settings['audio_mode'] = audio.audio_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()
    g_pool.capture.close()

    #shut down eye processes:
    stop_eye_process(0,blocking = True)
    stop_eye_process(1,blocking = True)

    #shut down laucher
    launcher_pipe.send("Exit")

    logger.info("Process Shutting down.")
Beispiel #43
0
    def __init__(
        self,
        g_pool,
        window_size=window_size_default,
        window_position=None,
        gui_scale=1.0,
        ui_config={},
    ):
        super().__init__(g_pool)

        self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128

        glfw.init()
        glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
        if g_pool.hide_ui:
            glfw.window_hint(glfw.VISIBLE, 0)  # hide window
        main_window = glfw.create_window(*window_size, "Pupil Service", None,
                                         None)

        window_position_manager = gl_utils.WindowPositionManager()
        window_position = window_position_manager.new_window_position(
            window=main_window,
            default_position=window_position_default,
            previous_position=window_position,
        )
        glfw.set_window_pos(main_window, *window_position)

        glfw.make_context_current(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        g_pool.gui = ui.UI()
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(0, 0),
                                           size=(0, 0),
                                           header_pos="headline")
        g_pool.gui.append(g_pool.menubar)

        # Callback functions
        def on_resize(window, w, h):
            # Always clear buffers on resize to make sure that there are no overlapping
            # artifacts from previous frames.
            gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT)
            gl_utils.glClearColor(0, 0, 0, 1)

            self.window_size = w, h
            self.content_scale = gl_utils.get_content_scale(window)
            g_pool.gui.scale = self.content_scale
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()

            # Needed, to update the window buffer while resizing
            self.update_ui()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
                window, x, y, cached_scale=None)
            g_pool.gui.update_mouse(x, y)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def set_window_size():
            # Get default window size
            f_width, f_height = window_size_default

            # Get current display scale factor
            content_scale = gl_utils.get_content_scale(main_window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
            display_scale_factor = content_scale / framebuffer_scale

            # Scale the capture frame size by display scale factor
            f_width *= display_scale_factor
            f_height *= display_scale_factor

            # Set the newly calculated size (scaled capture frame size + scaled icon bar width)
            glfw.set_window_size(main_window, int(f_width), int(f_height))

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.set_window_should_close(main_window, True)
            self.notify_all({"subject": "clear_settings_process.should_start"})
            self.notify_all({
                "subject": "service_process.should_start",
                "delay": 2.0
            })

        g_pool.menubar.append(ui.Button("Reset window size", set_window_size))

        pupil_remote_addr = "{}:{}".format(
            socket.gethostbyname(socket.gethostname()),
            g_pool.preferred_remote_port)
        g_pool.menubar.append(
            ui.Text_Input(
                "pupil_remote_addr",
                getter=lambda: pupil_remote_addr,
                setter=lambda x: None,
                label="Pupil Remote address",
            ))
        g_pool.menubar.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: self.start_stop_eye(0, alive),
                getter=lambda: g_pool.eye_procs_alive[0].value,
            ))
        g_pool.menubar.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: self.start_stop_eye(1, alive),
                getter=lambda: g_pool.eye_procs_alive[1].value,
            ))

        g_pool.menubar.append(
            ui.Info_Text("Service Version: {}".format(g_pool.version)))

        g_pool.menubar.append(
            ui.Button("Restart with default settings", reset_restart))

        # Register callbacks main_window
        glfw.set_framebuffer_size_callback(main_window, on_resize)
        glfw.set_key_callback(main_window, on_window_key)
        glfw.set_char_callback(main_window, on_window_char)
        glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
        glfw.set_cursor_pos_callback(main_window, on_pos)
        glfw.set_scroll_callback(main_window, on_scroll)
        g_pool.gui.configuration = ui_config
        gl_utils.basic_gl_setup()

        on_resize(g_pool.main_window, *glfw.get_framebuffer_size(main_window))
Beispiel #44
0
def main():

    # Callback functions
    def on_resize(window, w, h):
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        adjust_gl_view(w, h, window)
        norm_size = normalize((w, h), glfwGetWindowSize(window))
        fb_size = denormalize(norm_size, glfwGetFramebufferSize(window))
        atb.TwWindowSize(*map(int, fb_size))
        glfwMakeContextCurrent(active_window)
        for p in g.plugins:
            p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        if not atb.TwEventKeyboardGLFW(key, action):
            if action == GLFW_PRESS:
                pass

    def on_char(window, char):
        if not atb.TwEventCharGLFW(char, 1):
            pass

    def on_button(window, button, action, mods):
        if not atb.TwEventMouseButtonGLFW(button, action):
            pos = glfwGetCursorPos(window)
            pos = normalize(pos, glfwGetWindowSize(main_window))
            pos = denormalize(pos,
                              (frame.img.shape[1],
                               frame.img.shape[0]))  # Position in img pixels
            for p in g.plugins:
                p.on_click(pos, button, action)

    def on_pos(window, x, y):
        norm_pos = normalize((x, y), glfwGetWindowSize(window))
        fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window))
        if atb.TwMouseMotion(int(fb_x), int(fb_y)):
            pass

    def on_scroll(window, x, y):
        if not atb.TwMouseWheel(int(x)):
            pass

    def on_close(window):
        glfwSetWindowShouldClose(main_window, True)
        logger.debug('Process closing from window')

    try:
        rec_dir = sys.argv[1]
    except:
        #for dev, supply hardcoded dir:
        rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
        if os.path.isdir(rec_dir):
            logger.debug("Dev option: Using hadcoded data dir.")
        else:
            if getattr(sys, 'frozen', False):
                logger.warning(
                    "You did not supply a data directory when you called this script! \
                   \nPlease drag a Pupil recoding directory onto the launch icon."
                )
            else:
                logger.warning(
                    "You did not supply a data directory when you called this script! \
                       \nPlease supply a Pupil recoding directory as first arg when calling Pupil Player."
                )
            return

    if not is_pupil_rec_dir(rec_dir):
        logger.error(
            "You did not supply a dir with the required files inside.")
        return

    #backwards compatibility fn.
    patch_meta_info(rec_dir)

    #parse and load data folder info
    video_path = rec_dir + "/world.avi"
    timestamps_path = rec_dir + "/timestamps.npy"
    gaze_positions_path = rec_dir + "/gaze_positions.npy"
    meta_info_path = rec_dir + "/info.csv"

    #parse info.csv file
    with open(meta_info_path) as info:
        meta_info = dict(
            ((line.strip().split('\t')) for line in info.readlines()))
    rec_version = meta_info["Capture Software Version"]
    rec_version_float = int(
        filter(type(rec_version).isdigit,
               rec_version)[:3]) / 100.  #(get major,minor,fix of version)
    logger.debug("Recording version: %s , %s" %
                 (rec_version, rec_version_float))

    #load gaze information
    gaze_list = np.load(gaze_positions_path)
    timestamps = np.load(timestamps_path)

    #correlate data
    positions_by_frame = correlate_gaze(gaze_list, timestamps)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))

    def load(var_name, default):
        return session_settings.get(var_name, default)

    def save(var_name, var):
        session_settings[var_name] = var

    # Initialize capture
    cap = autoCreateCapture(video_path, timestamps=timestamps_path)

    if isinstance(cap, FakeCapture):
        logger.error("could not start capture.")
        return

    width, height = cap.get_size()

    # Initialize glfw
    glfwInit()
    main_window = glfwCreateWindow(
        width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
        rec_dir.split(os.path.sep)[-1], None, None)
    glfwMakeContextCurrent(main_window)

    # Register callbacks main_window
    glfwSetWindowSizeCallback(main_window, on_resize)
    glfwSetWindowCloseCallback(main_window, on_close)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)

    # create container for globally scoped varfs (within world)
    g = Temp()
    g.plugins = []
    g.play = False
    g.new_seek = True
    g.user_dir = user_dir
    g.rec_dir = rec_dir
    g.app = 'player'
    g.timestamps = timestamps
    g.positions_by_frame = positions_by_frame

    # helpers called by the main atb bar
    def update_fps():
        old_time, bar.timestamp = bar.timestamp, time()
        dt = bar.timestamp - old_time
        if dt:
            bar.fps.value += .1 * (1. / dt - bar.fps.value)

    def set_window_size(mode, data):
        width, height = cap.get_size()
        ratio = (1, .75, .5, .25)[mode]
        w, h = int(width * ratio), int(height * ratio)
        glfwSetWindowSize(main_window, w, h)
        data.value = mode  # update the bar.value

    def get_from_data(data):
        """
        helper for atb getter and setter use
        """
        return data.value

    def get_play():
        return g.play

    def set_play(value):
        g.play = value

    def next_frame():
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except FileSeekError:
            pass
        g.new_seek = True

    def prev_frame():
        try:
            cap.seek_to_frame(cap.get_frame_index() - 2)
        except FileSeekError:
            pass
        g.new_seek = True

    def open_plugin(selection, data):
        if plugin_by_index[selection] not in additive_plugins:
            for p in g.plugins:
                if isinstance(p, plugin_by_index[selection]):
                    return

        g.plugins = [p for p in g.plugins if p.alive]
        logger.debug('Open Plugin: %s' % name_by_index[selection])
        new_plugin = plugin_by_index[selection](g)
        g.plugins.append(new_plugin)
        g.plugins.sort(key=lambda p: p.order)

        if hasattr(new_plugin, 'init_gui'):
            new_plugin.init_gui()
        # save the value for atb bar
        data.value = selection

    def get_from_data(data):
        """
        helper for atb getter and setter use
        """
        return data.value

    atb.init()
    # add main controls ATB bar
    bar = atb.Bar(name="Controls",
                  label="Controls",
                  help="Scene controls",
                  color=(50, 50, 50),
                  alpha=100,
                  valueswidth=150,
                  text='light',
                  position=(10, 10),
                  refresh=.1,
                  size=(300, 160))
    bar.next_atb_pos = (10, 220)
    bar.fps = c_float(0.0)
    bar.timestamp = time()
    bar.window_size = c_int(load("window_size", 0))
    window_size_enum = atb.enum("Display Size", {
        "Full": 0,
        "Medium": 1,
        "Half": 2,
        "Mini": 3
    })
    bar.version = create_string_buffer(version, 512)
    bar.recording_version = create_string_buffer(rec_version, 512)
    bar.add_var("fps", bar.fps, step=1., readonly=True)
    bar._fps = c_float(cap.get_fps())
    bar.add_var("recoding fps", bar._fps, readonly=True)
    bar.add_var("display size",
                vtype=window_size_enum,
                setter=set_window_size,
                getter=get_from_data,
                data=bar.window_size)
    bar.add_var("play",
                vtype=c_bool,
                getter=get_play,
                setter=set_play,
                key="space")
    bar.add_button('step next', next_frame, key='right')
    bar.add_button('step prev', prev_frame, key='left')
    bar.add_var("frame index",
                vtype=c_int,
                getter=lambda: cap.get_frame_index() - 1)

    bar.plugin_to_load = c_int(0)
    plugin_type_enum = atb.enum("Plug In", index_by_name)
    bar.add_var("plugin",
                setter=open_plugin,
                getter=get_from_data,
                data=bar.plugin_to_load,
                vtype=plugin_type_enum)
    bar.add_var(
        "version of recording",
        bar.recording_version,
        readonly=True,
        help="version of the capture software used to make this recording")
    bar.add_var("version of player",
                bar.version,
                readonly=True,
                help="version of the Pupil Player")
    bar.add_button("exit", on_close, data=main_window, key="esc")

    #set the last saved window size
    set_window_size(bar.window_size.value, bar.window_size)
    on_resize(main_window, *glfwGetWindowSize(main_window))
    glfwSetWindowPos(main_window, 0, 0)

    #we always load these plugins
    g.plugins.append(
        Export_Launcher(g, data_dir=rec_dir, frame_count=len(timestamps)))
    g.plugins.append(Seek_Bar(g, capture=cap))
    g.trim_marks = Trim_Marks(g, capture=cap)
    g.plugins.append(g.trim_marks)

    #these are loaded based on user settings
    for initializer in load('plugins', []):
        name, args = initializer
        logger.debug("Loading plugin: %s with settings %s" % (name, args))
        try:
            p = plugin_by_name[name](g, **args)
            g.plugins.append(p)
        except:
            logger.warning("Plugin '%s' failed to load from settings file." %
                           name)

    if load('plugins', "_") == "_":
        #lets load some default if we dont have presets
        g.plugins.append(Scan_Path(g))
        g.plugins.append(Vis_Polyline(g))
        g.plugins.append(Vis_Circle(g))
        # g.plugins.append(Vis_Light_Points(g))

    #sort by exec order
    g.plugins.sort(key=lambda p: p.order)

    #init gui
    for p in g.plugins:
        if hasattr(p, 'init_gui'):
            p.init_gui()

    # gl_state settings
    basic_gl_setup()
    g.image_tex = create_named_texture((height, width, 3))

    while not glfwWindowShouldClose(main_window):

        update_fps()

        #grab new frame
        if g.play or g.new_seek:
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                #end of video logic: pause at last frame.
                g.play = False

            if g.new_seek:
                display_time = new_frame.timestamp
                g.new_seek = False

        frame = new_frame.copy()
        #new positons and events we make a deepcopy just like the image is a copy.
        current_pupil_positions = deepcopy(positions_by_frame[frame.index])
        events = []

        # allow each Plugin to do its work.
        for p in g.plugins:
            p.update(frame, current_pupil_positions, events)

        #check if a plugin need to be destroyed
        g.plugins = [p for p in g.plugins if p.alive]

        # render camera image
        glfwMakeContextCurrent(main_window)
        make_coord_system_norm_based()
        draw_named_texture(g.image_tex, frame.img)
        make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g.plugins:
            p.gl_display()

        #present frames at appropriate speed
        wait_time = frame.timestamp - display_time
        display_time = frame.timestamp
        try:
            spent_time = time() - timestamp
            sleep(wait_time - spent_time)
        except:
            pass
        timestamp = time()

        atb.draw()
        glfwSwapBuffers(main_window)
        glfwPollEvents()

    plugin_save = []
    for p in g.plugins:
        try:
            p_initializer = p.get_class_name(), p.get_init_dict()
            plugin_save.append(p_initializer)
        except AttributeError:
            #not all plugins need to be savable, they will not have the init dict.
            # any object without a get_init_dict method will throw this exception.
            pass

    # de-init all running plugins
    for p in g.plugins:
        p.alive = False
        #reading p.alive actually runs plug-in cleanup
        _ = p.alive

    save('plugins', plugin_save)
    save('window_size', bar.window_size.value)
    session_settings.close()

    cap.close()
    bar.destroy()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    logger.debug("Process done")
Beispiel #45
0
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport
        from ui_roi import UIRoi

        # monitoring
        import psutil

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, AV_Writer
        from ndsi import H264Writer
        from video_capture import source_classes
        from video_capture import manager_classes

        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy

        pupil_detectors = {
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D,
            Detector_Dummy.__name__: Detector_Dummy,
        }

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = "capture"
        g_pool.process = "eye{}".format(eye_id)
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor

            active_window = glfw.glfwGetCurrentContext()
            glfw.glfwMakeContextCurrent(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x *= hdpi_factor
            y *= hdpi_factor
            g_pool.gui.update_mouse(x, y)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, g_pool.capture.frame_size)
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            plugins = (g_pool.capture_manager, g_pool.capture)
            # call `on_drop` callbacks until a plugin indicates
            # that it has consumed the event (by returning True)
            any(p.on_drop(paths) for p in plugins)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
        )
        if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.iconified = False
        g_pool.capture = None
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get("flip", False)
        g_pool.display_mode = session_settings.get("display_mode", "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image": "Raw eye camera image. This uses the least amount of CPU power",
            "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        capture_manager_settings = session_settings.get(
            "capture_manager_settings", ("UVC_Manager", {})
        )

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__: c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](
            g_pool, **manager_settings
        )

        if eye_id == 0:
            cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"]
        else:
            cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"]

        # Initialize capture
        default_settings = (
            "UVC_Source",
            {"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120},
        )

        capture_source_settings = overwrite_cap_settings or session_settings.get(
            "capture_settings", default_settings
        )
        source_class_name, source_settings = capture_source_settings
        source_class_by_name = {c.__name__: c for c in source_classes}
        g_pool.capture = source_class_by_name[source_class_name](
            g_pool, **source_settings
        )
        assert g_pool.capture

        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get("roi")
        if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        pupil_detector_settings = session_settings.get("pupil_detector_settings", None)
        last_pupil_detector = pupil_detectors[
            session_settings.get("last_pupil_detector", Detector_2D.__name__)
        ]
        g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings)

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.deinit_ui()
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_ui()

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.glfwInit()
        title = "Pupil Capture - eye {}".format(eye_id)

        width, height = g_pool.capture.frame_size
        width *= 2
        height *= 2
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get("window_position", window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2],
                label="Interface Size",
            )
        )

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width *= 2
            f_height *= 2
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        def uroi_on_mouse_button(button, action, mods):
            if g_pool.display_mode == "roi":
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    x *= hdpi_factor
                    y *= hdpi_factor
                    pos = normalize((x, y), camera_render_size)
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(
                        pos, g_pool.capture.frame_size
                    )  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                        pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size
                    ):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            )
        )
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode]
        )

        general_settings.append(g_pool.display_mode_info)

        detector_selector = ui.Selector(
            "pupil_detector",
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Detector_Dummy, Detector_2D, Detector_3D],
            labels=["disabled", "C++ 2d detector", "C++ 3d detector"],
            label="Detection method",
        )
        general_settings.append(detector_selector)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)
        toggle_general_settings(False)

        g_pool.pupil_detector.init_ui()
        g_pool.capture.init_ui()
        g_pool.capture_manager.init_ui()
        g_pool.writer = None

        def replace_source(source_class_name, source_settings):
            g_pool.capture.deinit_ui()
            g_pool.capture.cleanup()
            g_pool.capture = source_class_by_name[source_class_name](
                g_pool, **source_settings
            )
            g_pool.capture.init_ui()
            if g_pool.writer:
                logger.info("Done recording.")
                try:
                    g_pool.writer.release()
                except RuntimeError:
                    logger.error("No eye video recorded")
                g_pool.writer = None

        g_pool.replace_source = replace_source  # for ndsi capture

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "set_detection_mapping_mode":
                    if notification["mode"] == "3d":
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    elif notification["mode"] == "2d":
                        if not isinstance(g_pool.pupil_detector, Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                    else:
                        if not isinstance(g_pool.pupil_detector, Detector_Dummy):
                            set_detector(Detector_Dummy)
                        detector_selector.read_only = True
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        record_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        logger.info("Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(
                            record_path, "eye{}.mp4".format(eye_id)
                        )
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(
                                video_path, g_pool.capture.frame_rate
                            )
                        elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = AV_Writer(
                                video_path, g_pool.capture.frame_rate
                            )
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify(
                        {
                            "subject": "meta.doc",
                            "actor": "eye{}".format(eye_id),
                            "doc": eye.__doc__,
                        }
                    )
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (
                    subject.startswith("start_eye_capture")
                    and notification["target"] == g_pool.process
                ):
                    replace_source(notification["name"], notification["args"])
                elif notification["subject"].startswith("pupil_detector.set_property"):
                    target_process = notification.get("target", g_pool.process)
                    should_apply = target_process == g_pool.process

                    if should_apply:
                        try:
                            property_name = notification["name"]
                            property_value = notification["value"]
                            if "2d" in notification["subject"]:
                                g_pool.pupil_detector.set_2d_detector_property(
                                    property_name, property_value
                                )
                            elif "3d" in notification["subject"]:
                                if not isinstance(g_pool.pupil_detector, Detector_3D):
                                    raise ValueError(
                                        "3d properties are only available"
                                        " if 3d detector is active"
                                    )
                                g_pool.pupil_detector.set_3d_detector_property(
                                    property_name, property_value
                                )
                            else:
                                raise KeyError(
                                    "Notification subject does not "
                                    "specifiy detector type."
                                )
                            logger.debug(
                                "`{}` property set to {}".format(
                                    property_name, property_value
                                )
                            )
                        except KeyError:
                            logger.error("Malformed notification received")
                            logger.debug(traceback.format_exc())
                        except (ValueError, TypeError):
                            logger.error("Invalid property or value")
                            logger.debug(traceback.format_exc())
                elif notification["subject"].startswith(
                    "pupil_detector.broadcast_properties"
                ):
                    target_process = notification.get("target", g_pool.process)
                    should_respond = target_process == g_pool.process
                    if should_respond:
                        props = g_pool.pupil_detector.get_detector_properties()
                        properties_broadcast = {
                            "subject": "pupil_detector.properties.{}".format(eye_id),
                            **props,  # add properties to broadcast
                        }
                        ipc_socket.notify(properties_broadcast)
                g_pool.capture.on_notify(notification)
                g_pool.capture_manager.on_notify(notification)

            # Get an image from the grabber
            event = {}
            g_pool.capture.recent_events(event)
            frame = event.get("frame")
            g_pool.capture_manager.recent_events(event)
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
                    f_height,
                    f_width,
                ):
                    g_pool.pupil_detector.on_resolution_change(
                        (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
                        g_pool.capture.frame_size,
                    )
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.format(
                                    type(frame), frame_publish_format
                                )
                            )
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send(
                            {
                                "topic": "frame.eye.{}".format(eye_id),
                                "width": frame.width,
                                "height": frame.height,
                                "index": frame.index,
                                "timestamp": frame.timestamp,
                                "format": frame_publish_format,
                                "__raw_data__": [data],
                            }
                        )

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                # pupil ellipse detection
                result = g_pool.pupil_detector.detect(
                    frame, g_pool.u_r, g_pool.display_mode == "algorithm"
                )
                if result is not None:
                    result["id"] = eye_id
                    result["topic"] = "pupil.{}".format(eye_id)
                    pupil_socket.send(result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    if frame:
                        # switch to work in normalized coordinate space
                        if g_pool.display_mode == "algorithm":
                            g_pool.image_tex.update_from_ndarray(frame.img)
                        elif g_pool.display_mode in ("camera_image", "roi"):
                            g_pool.image_tex.update_from_ndarray(frame.gray)
                        else:
                            pass
                    glViewport(0, 0, *camera_render_size)
                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    f_width, f_height = g_pool.capture.frame_size
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    if frame and result:
                        if result["method"] == "3d c++":
                            eye_ball = result["projected_sphere"]
                            try:
                                pts = cv2.ellipse2Poly(
                                    (
                                        int(eye_ball["center"][0]),
                                        int(eye_ball["center"][1]),
                                    ),
                                    (
                                        int(eye_ball["axes"][0] / 2),
                                        int(eye_ball["axes"][1] / 2),
                                    ),
                                    int(eye_ball["angle"]),
                                    0,
                                    360,
                                    8,
                                )
                            except ValueError as e:
                                pass
                            else:
                                draw_polyline(
                                    pts,
                                    2,
                                    RGBA(0.0, 0.9, 0.1, result["model_confidence"]),
                                )
                        if result["confidence"] > 0:
                            if "ellipse" in result:
                                pts = cv2.ellipse2Poly(
                                    (
                                        int(result["ellipse"]["center"][0]),
                                        int(result["ellipse"]["center"][1]),
                                    ),
                                    (
                                        int(result["ellipse"]["axes"][0] / 2),
                                        int(result["ellipse"]["axes"][1] / 2),
                                    ),
                                    int(result["ellipse"]["angle"]),
                                    0,
                                    360,
                                    15,
                                )
                                confidence = result["confidence"] * 0.7
                                draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence))
                                draw_points(
                                    [result["ellipse"]["center"]],
                                    size=20,
                                    color=RGBA(1.0, 0.0, 0.0, confidence),
                                    sharpness=1.0,
                                )

                    glViewport(0, 0, *camera_render_size)
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == "roi":
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    glViewport(0, 0, *window_size)
                    make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    unused_elements = g_pool.gui.update()
                    for butt in unused_elements.buttons:
                        uroi_on_mouse_button(*butt)

                    make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)

                    g_pool.pupil_detector.visualize()  # detector decides if we visualize or not

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer = None

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        # save session persistent settings
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["roi"] = g_pool.u_r.get()
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["capture_settings"] = (
            g_pool.capture.class_name,
            g_pool.capture.get_init_dict(),
        )
        session_settings["capture_manager_settings"] = (
            g_pool.capture_manager.class_name,
            g_pool.capture_manager.get_init_dict(),
        )
        session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
        session_settings["version"] = str(g_pool.version)
        session_settings[
            "last_pupil_detector"
        ] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            "pupil_detector_settings"
        ] = g_pool.pupil_detector.get_settings()

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        g_pool.capture.deinit_ui()
        g_pool.capture_manager.deinit_ui()
        g_pool.pupil_detector.deinit_ui()

        g_pool.pupil_detector.cleanup()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()

        glfw.glfwDestroyWindow(main_window)
        g_pool.gui.terminate()
        glfw.glfwTerminate()
        logger.info("Process shutting down.")
Beispiel #46
0
def world(g_pool, cap_src, cap_size):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from g_pool.pupil_queue
    Can run various plug-ins.
    """

    # Callback functions
    def on_resize(window, w, h):
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        norm_size = normalize((w, h), glfwGetWindowSize(window))
        fb_size = denormalize(norm_size, glfwGetFramebufferSize(window))
        atb.TwWindowSize(*map(int, fb_size))
        adjust_gl_view(w, h, window)
        glfwMakeContextCurrent(active_window)
        for p in g_pool.plugins:
            p.on_window_resize(window, w, h)

    def on_iconify(window, iconfied):
        if not isinstance(cap, FakeCapture):
            g_pool.update_textures.value = not iconfied

    def on_key(window, key, scancode, action, mods):
        if not atb.TwEventKeyboardGLFW(key, action):
            if action == GLFW_PRESS:
                if key == GLFW_KEY_ESCAPE:
                    on_close(window)

    def on_char(window, char):
        if not atb.TwEventCharGLFW(char, 1):
            pass

    def on_button(window, button, action, mods):
        if not atb.TwEventMouseButtonGLFW(button, action):
            pos = glfwGetCursorPos(window)
            pos = normalize(pos, glfwGetWindowSize(world_window))
            pos = denormalize(pos,
                              (frame.img.shape[1],
                               frame.img.shape[0]))  # Position in img pixels
            for p in g_pool.plugins:
                p.on_click(pos, button, action)

    def on_pos(window, x, y):
        norm_pos = normalize((x, y), glfwGetWindowSize(window))
        fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window))
        if atb.TwMouseMotion(int(fb_x), int(fb_y)):
            pass

    def on_scroll(window, x, y):
        if not atb.TwMouseWheel(int(x)):
            pass

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_world'))

    def load(var_name, default):
        return session_settings.get(var_name, default)

    def save(var_name, var):
        session_settings[var_name] = var

    # Initialize capture
    cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)

    # Get an image from the grabber
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return
    height, width = frame.img.shape[:2]

    # load last calibration data
    try:
        pt_cloud = np.load(os.path.join(g_pool.user_dir, 'cal_pt_cloud.npy'))
        logger.debug("Using calibration found in %s" % g_pool.user_dir)
        map_pupil = calibrate.get_map_from_cloud(pt_cloud, (width, height))
    except:
        logger.debug("No calibration found.")

        def map_pupil(vector):
            """ 1 to 1 mapping """
            return vector

    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.plugins = []
    g_pool.map_pupil = map_pupil
    g_pool.update_textures = c_bool(1)
    if isinstance(cap, FakeCapture):
        g_pool.update_textures.value = False
    g_pool.capture = cap

    g_pool.rec_name = recorder.get_auto_name()

    # helpers called by the main atb bar
    def update_fps():
        old_time, bar.timestamp = bar.timestamp, time()
        dt = bar.timestamp - old_time
        if dt:
            bar.fps.value += .05 * (1. / dt - bar.fps.value)

    def set_window_size(mode, data):
        height, width = frame.img.shape[:2]
        ratio = (1, .75, .5, .25)[mode]
        w, h = int(width * ratio), int(height * ratio)
        glfwSetWindowSize(world_window, w, h)
        data.value = mode  # update the bar.value

    def get_from_data(data):
        """
        helper for atb getter and setter use
        """
        return data.value

    def set_rec_name(val):
        if not val.value:
            g_pool.rec_name = recorder.get_auto_name()
        else:
            g_pool.rec_name = val.value

    def get_rec_name():
        return create_string_buffer(g_pool.rec_name, 512)

    def open_calibration(selection, data):
        # prepare destruction of current ref_detector... and remove it
        for p in g_pool.plugins:
            if isinstance(p, calibration_routines.detector_by_index):
                p.alive = False
        g_pool.plugins = [p for p in g_pool.plugins if p.alive]

        new_ref_detector = calibration_routines.detector_by_index[selection](
            g_pool, atb_pos=bar.next_atb_pos)
        g_pool.plugins.append(new_ref_detector)
        g_pool.plugins.sort(key=lambda p: p.order)

        # save the value for atb bar
        data.value = selection

    def toggle_record_video():
        for p in g_pool.plugins:
            if isinstance(p, recorder.Recorder):
                p.alive = False
                return

        new_plugin = recorder.Recorder(g_pool, g_pool.rec_name, bar.fps.value,
                                       frame.img.shape, bar.record_eye.value,
                                       g_pool.eye_tx, bar.audio.value)
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def toggle_show_calib_result():
        for p in g_pool.plugins:
            if isinstance(p, Show_Calibration):
                p.alive = False
                return

        new_plugin = Show_Calibration(g_pool, frame.img.shape)
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def toggle_server():
        for p in g_pool.plugins:
            if isinstance(p, Pupil_Server):
                p.alive = False
                return

        new_plugin = Pupil_Server(g_pool, (10, 300))
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def toggle_remote():
        for p in g_pool.plugins:
            if isinstance(p, Pupil_Remote):
                p.alive = False
                return

        new_plugin = Pupil_Remote(g_pool, (10, 360), on_char)
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def toggle_ar():
        for p in g_pool.plugins:
            if isinstance(p, Marker_Detector):
                p.alive = False
                return

        new_plugin = Marker_Detector(g_pool, (10, 400))
        g_pool.plugins.append(new_plugin)
        g_pool.plugins.sort(key=lambda p: p.order)

    def reset_timebase():
        #the last frame from worldcam will be t0
        g_pool.timebase.value = g_pool.capure.get_now()
        logger.info(
            "New timebase set to %s all timestamps will count from here now." %
            g_pool.timebase.value)

    atb.init()
    # add main controls ATB bar
    bar = atb.Bar(name="World",
                  label="Controls",
                  help="Scene controls",
                  color=(50, 50, 50),
                  alpha=100,
                  valueswidth=150,
                  text='light',
                  position=(10, 10),
                  refresh=.3,
                  size=(300, 200))
    bar.next_atb_pos = (10, 220)
    bar.fps = c_float(0.0)
    bar.timestamp = time()
    bar.calibration_type = c_int(load("calibration_type", 0))
    bar.record_eye = c_bool(load("record_eye", 0))
    bar.audio = c_int(load("audio", -1))
    bar.window_size = c_int(load("window_size", 0))
    window_size_enum = atb.enum("Display Size", {
        "Full": 0,
        "Medium": 1,
        "Half": 2,
        "Mini": 3
    })
    calibrate_type_enum = atb.enum("Calibration Method",
                                   calibration_routines.index_by_name)
    audio_enum = atb.enum("Audio Input", dict(Audio_Input_List()))
    bar.version = create_string_buffer(g_pool.version, 512)
    bar.add_var(
        "fps",
        bar.fps,
        step=1.,
        readonly=True,
        help=
        "Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate."
    )
    bar.add_var(
        "display size",
        vtype=window_size_enum,
        setter=set_window_size,
        getter=get_from_data,
        data=bar.window_size,
        help="Resize the world window. This has no effect on the actual image."
    )
    bar.add_var("calibration method",
                setter=open_calibration,
                getter=get_from_data,
                data=bar.calibration_type,
                vtype=calibrate_type_enum,
                group="Calibration",
                help="Please choose your desired calibration method.")
    bar.add_button("show calibration result",
                   toggle_show_calib_result,
                   group="Calibration",
                   help="Click to show calibration result.")
    bar.add_var("session name",
                create_string_buffer(512),
                getter=get_rec_name,
                setter=set_rec_name,
                group="Recording",
                help="Give your recording session a custom name.")
    bar.add_button("record",
                   toggle_record_video,
                   key="r",
                   group="Recording",
                   help="Start/Stop Recording")
    bar.add_var("record eye",
                bar.record_eye,
                group="Recording",
                help="check to save raw video of eye")
    bar.add_var("record audio",
                bar.audio,
                vtype=audio_enum,
                group="Recording",
                help="Select from audio recording options.")
    bar.add_button(
        "start/stop marker tracking",
        toggle_ar,
        key="x",
        help="find markers in scene to map gaze onto referace surfaces")
    bar.add_button(
        "start/stop server",
        toggle_server,
        key="s",
        help=
        "the server broadcasts pupil and gaze positions locally or via network"
    )
    bar.add_button("start/stop remote",
                   toggle_remote,
                   key="w",
                   help="remote allows seding commad to pupil via network")
    bar.add_button(
        "set timebase to now",
        reset_timebase,
        help="this button allows the timestamps to count from now on.",
        key="t")
    bar.add_var(
        "update screen",
        g_pool.update_textures,
        help=
        "if you dont need to see the camera image updated, you can turn this of to reduce CPU load."
    )
    bar.add_separator("Sep1")
    bar.add_var("version", bar.version, readonly=True)
    bar.add_var("exit", g_pool.quit)

    # add uvc camera controls ATB bar
    cap.create_atb_bar(pos=(320, 10))

    # Initialize glfw
    glfwInit()
    world_window = glfwCreateWindow(width, height, "World", None, None)
    glfwMakeContextCurrent(world_window)

    # Register callbacks world_window
    glfwSetWindowSizeCallback(world_window, on_resize)
    glfwSetWindowCloseCallback(world_window, on_close)
    glfwSetWindowIconifyCallback(world_window, on_iconify)
    glfwSetKeyCallback(world_window, on_key)
    glfwSetCharCallback(world_window, on_char)
    glfwSetMouseButtonCallback(world_window, on_button)
    glfwSetCursorPosCallback(world_window, on_pos)
    glfwSetScrollCallback(world_window, on_scroll)

    #set the last saved window size
    set_window_size(bar.window_size.value, bar.window_size)
    on_resize(world_window, *glfwGetWindowSize(world_window))
    glfwSetWindowPos(world_window, 0, 0)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = create_named_texture(frame.img)
    # refresh speed settings
    glfwSwapInterval(0)

    #load calibration plugin
    open_calibration(bar.calibration_type.value, bar.calibration_type)

    #load gaze_display plugin
    g_pool.plugins.append(Display_Recent_Gaze(g_pool))

    # Event loop
    while not g_pool.quit.value:

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break

        update_fps()

        #a container that allows plugins to post and read events
        events = []

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            if p['norm_pupil'] is None:
                p['norm_gaze'] = None
            else:
                p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil'])
            recent_pupil_positions.append(p)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, recent_pupil_positions, events)

        #check if a plugin need to be destroyed
        g_pool.plugins = [p for p in g_pool.plugins if p.alive]

        # render camera image
        glfwMakeContextCurrent(world_window)

        make_coord_system_norm_based()
        if g_pool.update_textures.value:
            draw_named_texture(g_pool.image_tex, frame.img)
        else:
            draw_named_texture(g_pool.image_tex)
        make_coord_system_pixel_based(frame.img.shape)

        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        atb.draw()
        glfwSwapBuffers(world_window)
        glfwPollEvents()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
        #reading p.alive actually runs plug-in cleanup
        _ = p.alive

    save('window_size', bar.window_size.value)
    save('calibration_type', bar.calibration_type.value)
    save('record_eye', bar.record_eye.value)
    save('audio', bar.audio.value)
    session_settings.close()

    cap.close()
    atb.terminate()
    glfwDestroyWindow(world_window)
    glfwTerminate()
    logger.debug("Process done")
Beispiel #47
0
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
    hide_ui=False,
    debug=False,
    pub_socket_hwm=None,
    parent_application="capture",
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
        ``eye_process.should_stop``: Stops the eye process
        ``recording.started``: Starts recording eye video
        ``recording.stopped``: Stops recording eye video
        ``frame_publishing.started``: Starts frame publishing
        ``frame_publishing.stopped``: Stops frame publishing
        ``start_eye_plugin``: Start plugins in eye process

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url, pub_socket_hwm)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        from OpenGL.GL import GL_COLOR_BUFFER_BIT

        # display
        import glfw
        from gl_utils import GLFWErrorReporting

        GLFWErrorReporting.set_default()

        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import Named_Texture
        import gl_utils
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport

        # monitoring
        import psutil

        # Plug-ins
        from plugin import Plugin_List

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import parse_version
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError
        from ndsi import H264Writer
        from video_capture import source_classes, manager_classes
        from roi import Roi

        from background_helper import IPC_Logging_Task_Proxy
        from pupil_detector_plugins import available_detector_plugins, EVENT_KEY

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            # NOTE: Interrupt is handled in world/service/player which are responsible for
            # shutting down the eye process properly

        signal.signal(signal.SIGINT, interrupt_handler)

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        content_scale = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.debug = debug
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = parent_application
        g_pool.eye_id = eye_id
        g_pool.process = f"eye{eye_id}"
        g_pool.timebase = timebase
        g_pool.camera_render_size = None

        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_socket
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        def load_runtime_pupil_detection_plugins():
            from plugin import import_runtime_plugins
            from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin

            plugins_path = os.path.join(g_pool.user_dir, "plugins")

            for plugin in import_runtime_plugins(plugins_path):
                if not isinstance(plugin, type):
                    continue
                if not issubclass(plugin, PupilDetectorPlugin):
                    continue
                if plugin is PupilDetectorPlugin:
                    continue
                yield plugin

        available_detectors = available_detector_plugins()
        runtime_detectors = list(load_runtime_pupil_detection_plugins())
        plugins = (
            manager_classes
            + source_classes
            + available_detectors
            + runtime_detectors
            + [Roi]
        )
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        preferred_names = [
            f"Pupil Cam3 ID{eye_id}",
            f"Pupil Cam2 ID{eye_id}",
            f"Pupil Cam1 ID{eye_id}",
        ]
        if eye_id == 0:
            preferred_names += ["HD-6000"]

        default_capture_name = "UVC_Source"
        default_capture_settings = {
            "preferred_names": preferred_names,
            "frame_size": (192, 192),
            "frame_rate": 120,
        }

        default_plugins = [
            # TODO: extend with plugins
            (default_capture_name, default_capture_settings),
            ("UVC_Manager", {}),
            *[(p.__name__, {}) for p in available_detectors],
            ("NDSI_Manager", {}),
            ("HMD_Streaming_Manager", {}),
            ("File_Manager", {}),
            ("Roi", {}),
        ]

        def consume_events_and_render_buffer():
            glfw.make_context_current(main_window)
            clear_gl_screen()

            if all(c > 0 for c in g_pool.camera_render_size):
                glViewport(0, 0, *g_pool.camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

            glViewport(0, 0, *window_size)
            # render graphs
            fps_graph.draw()
            cpu_graph.draw()

            # render GUI
            try:
                clipboard = glfw.get_clipboard_string(main_window).decode()
            except (AttributeError, glfw.GLFWError):
                # clipboard is None, might happen on startup
                clipboard = ""
            g_pool.gui.update_clipboard(clipboard)
            user_input = g_pool.gui.update()
            if user_input.clipboard != clipboard:
                # only write to clipboard if content changed
                glfw.set_clipboard_string(main_window, user_input.clipboard)

            for button, action, mods in user_input.buttons:
                x, y = glfw.get_cursor_pos(main_window)
                pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                    main_window, x, y, cached_scale=None
                )
                pos = normalize(pos, g_pool.camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                # Position in img pixels
                pos = denormalize(pos, g_pool.capture.frame_size)

                for plugin in g_pool.plugins:
                    if plugin.on_click(pos, button, action):
                        break

            for key, scancode, action, mods in user_input.keys:
                for plugin in g_pool.plugins:
                    if plugin.on_key(key, scancode, action, mods):
                        break

            for char_ in user_input.chars:
                for plugin in g_pool.plugins:
                    if plugin.on_char(char_):
                        break

            # update screen
            glfw.swap_buffers(main_window)

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal content_scale

            is_minimized = bool(glfw.get_window_attrib(window, glfw.ICONIFIED))

            if is_minimized:
                return

            # Always clear buffers on resize to make sure that there are no overlapping
            # artifacts from previous frames.
            gl_utils.glClear(GL_COLOR_BUFFER_BIT)
            gl_utils.glClearColor(0, 0, 0, 1)

            active_window = glfw.get_current_context()
            glfw.make_context_current(window)
            content_scale = gl_utils.get_content_scale(window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(window)
            g_pool.gui.scale = content_scale
            window_size = w, h
            g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = content_scale
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.make_context_current(active_window)

            # Minimum window size required, otherwise parts of the UI can cause openGL
            # issues with permanent effects. Depends on the content scale, which can
            # potentially be dynamically modified, so we re-adjust the size limits every
            # time here.
            min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale)
            glfw.set_window_size_limits(
                window,
                min_size,
                min_size,
                glfw.DONT_CARE,
                glfw.DONT_CARE,
            )

            # Needed, to update the window buffer while resizing
            consume_events_and_render_buffer()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
                window, x, y, cached_scale=None
            )
            g_pool.gui.update_mouse(x, y)

            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            if g_pool.flip:
                pos = 1 - pos[0], 1 - pos[1]
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)

            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, paths):
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
        )
        if parse_version(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        camera_is_physically_flipped = eye_id == 0
        g_pool.iconified = False
        g_pool.capture = None
        g_pool.flip = session_settings.get("flip", camera_is_physically_flipped)
        g_pool.display_mode = session_settings.get("display_mode", "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image": "Raw eye camera image. This uses the least amount of CPU power",
            "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.init()
        glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
        if hide_ui:
            glfw.window_hint(glfw.VISIBLE, 0)  # hide window
        title = "Pupil Capture - eye {}".format(eye_id)

        # Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions.
        # As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3
        # cameras, we adjust the default eye window size to a 1:1 content aspect ratio.
        # The size of 500 was chosen s.t. the menu still fits.
        default_window_size = 500 + icon_bar_width, 500
        width, height = session_settings.get("window_size", default_window_size)

        main_window = glfw.create_window(width, height, title, None, None)

        window_position_manager = gl_utils.WindowPositionManager()
        window_pos = window_position_manager.new_window_position(
            window=main_window,
            default_position=window_position_default,
            previous_position=session_settings.get("window_position", None),
        )
        glfw.set_window_pos(main_window, window_pos[0], window_pos[1])

        glfw.make_context_current(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")

        def set_window_size():
            # Get current capture frame size
            f_width, f_height = g_pool.capture.frame_size
            # Eye camera resolutions are too small to be used as default window sizes.
            # We use double their size instead.
            frame_scale_factor = 2
            f_width *= frame_scale_factor
            f_height *= frame_scale_factor

            # Get current display scale factor
            content_scale = gl_utils.get_content_scale(main_window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
            display_scale_factor = content_scale / framebuffer_scale

            # Scale the capture frame size by display scale factor
            f_width *= display_scale_factor
            f_height *= display_scale_factor

            # Increas the width to account for the added scaled icon bar width
            f_width += icon_bar_width * display_scale_factor

            # Set the newly calculated size (scaled capture frame size + scaled icon bar width)
            glfw.set_window_size(main_window, int(f_width), int(f_height))

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            )
        )
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode]
        )

        general_settings.append(g_pool.display_mode_info)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        plugins_to_load = session_settings.get("loaded_plugins", default_plugins)
        if overwrite_cap_settings:
            # Ensure that overwrite_cap_settings takes preference over source plugins
            # with incorrect settings that were loaded from session settings.
            plugins_to_load.append(overwrite_cap_settings)

        # Add runtime plugins to the list of plugins to load with default arguments,
        # if not already restored from session settings
        plugins_to_load_names = set(name for name, _ in plugins_to_load)
        for runtime_detector in runtime_detectors:
            runtime_name = runtime_detector.__name__
            if runtime_name not in plugins_to_load_names:
                plugins_to_load.append((runtime_name, {}))

        g_pool.plugins = Plugin_List(g_pool, plugins_to_load)

        if not g_pool.capture:
            # Make sure we always have a capture running. Important if there was no
            # capture stored in session settings.
            g_pool.plugins.add(
                g_pool.plugin_by_name[default_capture_name], default_capture_settings
            )

        toggle_general_settings(True)

        g_pool.writer = None
        g_pool.rec_path = None

        # Register callbacks main_window
        glfw.set_framebuffer_size_callback(main_window, on_resize)
        glfw.set_window_iconify_callback(main_window, on_iconify)
        glfw.set_key_callback(main_window, on_window_key)
        glfw.set_char_callback(main_window, on_window_char)
        glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
        glfw.set_cursor_pos_callback(main_window, on_pos)
        glfw.set_scroll_callback(main_window, on_scroll)
        glfw.set_drop_callback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})
        # If previously selected plugin was not loaded this time, we will have an
        # expanded menubar without any menu selected. We need to ensure the menubar is
        # collapsed in this case.
        if all(submenu.collapsed for submenu in g_pool.menubar.elements):
            g_pool.menubar.collapsed = True

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.get_framebuffer_size(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        if platform.system() == "Darwin":
            # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of
            # occluded windows, based on the swap interval value. This causes an FPS drop
            # and leads to problems when recording. To side-step this behaviour, the swap
            # interval is set to zero.
            #
            # Read more about window occlusion on macOS here:
            # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html
            glfw.swap_interval(0)

        # Event loop
        window_should_close = False
        while not window_should_close:

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        g_pool.rec_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        start_time_synced = notification["start_time_synced"]
                        logger.info(f"Will save eye video to: {g_pool.rec_path}")
                        video_path = os.path.join(
                            g_pool.rec_path, "eye{}.mp4".format(eye_id)
                        )
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(video_path, start_time_synced)
                        elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = MPEG_Writer(video_path, start_time_synced)
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        else:
                            # TODO: wrap recording logic into plugin
                            g_pool.capture.intrinsics.save(
                                g_pool.rec_path, custom_name=f"eye{eye_id}"
                            )
                        finally:
                            g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify(
                        {
                            "subject": "meta.doc",
                            "actor": "eye{}".format(eye_id),
                            "doc": eye.__doc__,
                        }
                    )
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (
                    subject.startswith("start_eye_plugin")
                    and notification["target"] == g_pool.process
                ):
                    try:
                        g_pool.plugins.add(
                            g_pool.plugin_by_name[notification["name"]],
                            notification.get("args", {}),
                        )
                    except KeyError as err:
                        logger.error(f"Attempt to load unknown plugin: {err}")
                elif (
                    subject.startswith("stop_eye_plugin")
                    and notification["target"] == g_pool.process
                ):
                    try:
                        plugin_to_stop = g_pool.plugin_by_name[notification["name"]]
                    except KeyError as err:
                        logger.error(f"Attempt to load unknown plugin: {err}")
                    else:
                        plugin_to_stop.alive = False
                        g_pool.plugins.clean()

                for plugin in g_pool.plugins:
                    plugin.on_notify(notification)

            event = {}
            for plugin in g_pool.plugins:
                plugin.recent_events(event)

            frame = event.get("frame")
            if frame:
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.format(
                                    type(frame), frame_publish_format
                                )
                            )
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send(
                            {
                                "topic": "frame.eye.{}".format(eye_id),
                                "width": frame.width,
                                "height": frame.height,
                                "index": frame.index,
                                "timestamp": frame.timestamp,
                                "format": frame_publish_format,
                                "__raw_data__": [data],
                            }
                        )

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    try:
                        g_pool.writer.write_video_frame(frame)
                    except NonMonotonicTimestampError as e:
                        logger.error(
                            "Recorder received non-monotonic timestamp!"
                            " Stopping the recording!"
                        )
                        logger.debug(str(e))
                        ipc_socket.notify({"subject": "recording.should_stop"})
                        ipc_socket.notify(
                            {"subject": "recording.should_stop", "remote_notify": "all"}
                        )

                for result in event.get(EVENT_KEY, ()):
                    pupil_socket.send(result)

            # GL drawing
            if window_should_update():
                cpu_graph.update()
                if is_window_visible(main_window):
                    consume_events_and_render_buffer()
                glfw.poll_events()
                window_should_close = glfw.window_should_close(main_window)

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer.release()
            g_pool.writer = None

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        # save session persistent settings
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)

        if not hide_ui:
            glfw.restore_window(main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.get_window_pos(main_window)
            session_window_size = glfw.get_window_size(main_window)
            if 0 not in session_window_size:
                f_width, f_height = session_window_size
                if platform.system() in ("Windows", "Linux"):
                    # Store unscaled window size as the operating system will scale the
                    # windows appropriately during launch on Windows and Linux.
                    f_width, f_height = (
                        f_width / content_scale,
                        f_height / content_scale,
                    )
                session_settings["window_size"] = int(f_width), int(f_height)

        session_settings.close()

        for plugin in g_pool.plugins:
            plugin.alive = False
        g_pool.plugins.clean()

    glfw.destroy_window(main_window)
    g_pool.gui.terminate()
    glfw.terminate()
    logger.info("Process shutting down.")
Beispiel #48
0
def session(rec_dir):

    # Callback functions
    def on_resize(window,w, h):
        g_pool.gui.update_window(w,h)
        g_pool.gui.collect_menus()
        graph.adjust_size(w,h)
        adjust_gl_view(w,h)
        for p in g_pool.plugins:
            p.on_window_resize(window,w,h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)

    def on_button(window,button, action, mods):
        g_pool.gui.update_button(button,action,mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos,glfwGetWindowSize(window))
        pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos,button,action)

    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*y_scroll_factor)


    def on_drop(window,count,paths):
        for x in range(count):
            new_rec_dir =  paths[x]
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '%s'"%new_rec_dir)
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window,True)
            else:
                logger.error("'%s' is not a valid pupil recording"%new_rec_dir)




    tick = delta_t()
    def get_dt():
        return next(tick)


    video_path = [f for f in glob(os.path.join(rec_dir,"world.*")) if f[-3:] in ('mp4','mkv','avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    #parse info.csv file
    meta_info_path = os.path.join(rec_dir,"info.csv")
    with open(meta_info_path) as info:
        meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) )

    rec_version = read_rec_version(meta_info)
    if rec_version >= VersionFormat('0.7.4'):
        pass
    if rec_version >= VersionFormat('0.7.3'):
        update_recording_0v73_to_current(rec_dir)
    elif rec_version >= VersionFormat('0.5'):
        update_recording_0v5_to_current(rec_dir)
    elif rec_version >= VersionFormat('0.4'):
        update_recording_0v4_to_current(rec_dir)
    elif rec_version >= VersionFormat('0.3'):
        update_recording_0v3_to_current(rec_dir)
        timestamps_path = os.path.join(rec_dir, "timestamps.npy")
    else:
        logger.Error("This recording is to old. Sorry.")
        return


    timestamps = np.load(timestamps_path)
    # Initialize capture
    cap = File_Capture(video_path,timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
    if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file):
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    width,height = session_settings.get('window_size',cap.frame_size)
    window_pos = session_settings.get('window_position',(0,0))
    main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.binocular = meta_info.get('Eye Mode','monocular') == 'binocular'
    g_pool.version = get_version(version_file)
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.rec_version = rec_version
    g_pool.meta_info = meta_info
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list,g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list,g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] #populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except FileSeekError:
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index()-2)
        except FileSeekError:
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def open_plugin(plugin):
        if plugin ==  "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = slice(g_pool.trim_marks.in_mark,g_pool.trim_marks.out_mark)
        export_dir = os.path.join(g_pool.rec_dir,'exports','%s-%s'%(export_range.start,export_range.stop))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                logger.warning("Previous export for range [%s-%s] already exsits - overwriting."%(export_range.start,export_range.stop))
        else:
            logger.info('Created export dir at "%s"'%export_dir)

        notification = {'subject':'should_export','range':export_range,'export_dir':export_dir}
        g_pool.notifications.append(notification)

    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.main_menu = ui.Growing_Menu("Settings",pos=(-350,20),size=(300,400))
    g_pool.main_menu.append(ui.Button("Close Pupil Player",lambda:glfwSetWindowShouldClose(main_window,True)))
    g_pool.main_menu.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=0.75,max=2.5,label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version))
    g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version))
    g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
                                        setter= open_plugin, getter = lambda: "Select to load"))
    g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins))
    g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.frame_size[0],cap.frame_size[1])) )
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
    g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE)
    g_pool.play_button.on_color[:] = (0,1.,.0,.8)
    g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT)
    g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT)
    g_pool.export_button = ui.Thumb('export',getter = lambda: False, setter = do_export, hotkey='e')
    g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button,g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)


    #we always load these plugins
    system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})]
    default_plugins = [('Log_Display',{}),('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Video_Export_Launcher',{})]
    previous_plugins = session_settings.get('loaded_plugins',default_plugins)
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins)


    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window,on_resize)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)
    glfwSetDropCallback(main_window,on_drop)
    #trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    g_pool.gui.configuration = session_settings.get('ui_config',{})

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = cap.get_timestamp()-.03

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260,110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    while not glfwWindowShouldClose(main_window):


        #grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame_nowait()
            except EndofVideoFileError:
                #end of video logic: pause at last frame.
                g_pool.play=False
            update_graph = True
        else:
            update_graph = False


        frame = new_frame.copy()
        events = {}
        #report time between now and the last loop interation
        events['dt'] = get_dt()
        #new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            #update performace graphs
            for p in  events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts != t:
                dt,ts = t-ts,t
            fps_graph.add(1./dt)

            g_pool.play_button.status_text = str(frame.index)
        #always update the CPU graph
        cpu_graph.update()


        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        graph.pop_view()
        g_pool.gui.update()

        #present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.close()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
Beispiel #49
0
def show_no_rec_window():
    from pyglui.pyfontstash import fontstash
    from pyglui.ui import get_roboto_font_path

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x]
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '%s'" % new_rec_dir)
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'%s' is not a valid pupil recording" %
                             new_rec_dir)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if session_settings.get("version",
                            VersionFormat('0.0')) < get_version(version_file):
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()
    w, h = session_settings.get('window_size', (1280, 720))
    window_pos = session_settings.get('window_position', (0, 0))

    glfwWindowHint(GLFW_RESIZABLE, 0)
    window = glfwCreateWindow(w, h, 'Pupil Player')
    glfwWindowHint(GLFW_RESIZABLE, 1)

    glfwMakeContextCurrent(window)
    glfwSetWindowPos(window, window_pos[0], window_pos[1])
    glfwSetDropCallback(window, on_drop)

    adjust_gl_view(w, h)
    glfont = fontstash.Context()
    glfont.add_font('roboto', get_roboto_font_path())
    glfont.set_align_string(v_align="center", h_align="middle")
    glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
    basic_gl_setup()
    glClearColor(0.5, .5, 0.5, 0.0)
    text = 'Drop a recording directory onto this window.'
    tip = '(Tip: You can drop a recording directory onto the app icon.)'
    # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
    while not glfwWindowShouldClose(window):
        clear_gl_screen()
        glfont.set_blur(10.5)
        glfont.set_color_float((0.0, 0.0, 0.0, 1.))
        glfont.set_size(w / 25.)
        glfont.draw_text(w / 2, .3 * h, text)
        glfont.set_size(w / 30.)
        glfont.draw_text(w / 2, .4 * h, tip)
        glfont.set_blur(0.96)
        glfont.set_color_float((1., 1., 1., 1.))
        glfont.set_size(w / 25.)
        glfont.draw_text(w / 2, .3 * h, text)
        glfont.set_size(w / 30.)
        glfont.draw_text(w / 2, .4 * h, tip)
        glfwSwapBuffers(window)
        glfwPollEvents()

    session_settings['window_position'] = glfwGetWindowPos(window)
    session_settings['version'] = get_version(version_file)
    session_settings.close()
    del glfont
    glfwDestroyWindow(window)
Beispiel #50
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime

    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        # display
        import glfw

        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils

        # capture
        from video_capture import File_Source

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation

        # from vis_scan_path import Vis_Scan_Path
        from seek_control import Seek_Control
        from surface_tracker import Surface_Tracker_Offline

        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from eye_movement import Offline_Eye_Movement_Detector
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producer.gaze_from_recording import GazeFromRecording
        from gaze_producer.gaze_from_offline_calibration import (
            GazeFromOfflineCalibration,
        )
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from video_export.plugins.imotions_exporter import iMotions_Exporter
        from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
        from video_export.plugins.world_video_exporter import World_Video_Exporter
        from video_capture import File_Source
        from video_overlay.plugins import Video_Overlay, Eye_Overlay

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.23"
        ), "pyglui out of date, please upgrade to newest version"

        runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins"))
        system_plugins = [
            Log_Display,
            Seek_Control,
            Plugin_Manager,
            System_Graphs,
            System_Timelines,
            Audio_Playback,
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Eye_Overlay,
            Video_Overlay,
            # Vis_Scan_Path,
            Offline_Fixation_Detector,
            Offline_Eye_Movement_Detector,
            Offline_Blink_Detection,
            Surface_Tracker_Offline,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            GazeFromRecording,
            GazeFromOfflineCalibration,
            World_Video_Exporter,
            iMotions_Exporter,
            Eye_Video_Exporter,
        ] + runtime_plugins

        if platform.system() != "Windows":
            # Head pose tracking is currently not available on Windows
            from head_pose_tracker.offline_head_pose_tracker import (
                Offline_Head_Pose_Tracker,
            )

            user_plugins.append(Offline_Head_Pose_Tracker)

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for path in paths:
                if pm.is_pupil_rec_dir(path):
                    _restart_with_recording(path)
                    return
            # call `on_drop` callbacks until a plugin indicates
            # that it has consumed the event (by returning True)
            any(p.on_drop(paths) for p in g_pool.plugins)

        def _restart_with_recording(rec_dir):
            logger.debug("Starting new session with '{}'".format(rec_dir))
            ipc_pub.notify(
                {"subject": "player_drop_process.should_start", "rec_dir": rec_dir}
            )
            glfw.glfwSetWindowShouldClose(g_pool.main_window, True)

        tick = delta_t()

        def get_dt():
            return next(tick)

        meta_info = pm.load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info("Application Version: {}".format(app_version))
        logger.info("System Info: {}".format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = SimpleNamespace()
        g_pool.app = "player"
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
        video_path = [
            f
            for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in valid_ext
        ][0]
        File_Source(
            g_pool,
            timing="external",
            source_path=video_path,
            buffered_decoding=True,
            fill_gaps=True,
        )

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player")
        )
        if VersionFormat(session_settings.get("version", "0.0")) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        window_pos = session_settings.get("window_position", window_position_default)
        window_name = "Pupil Player: {} - {}".format(
            meta_info["Recording Name"], os.path.split(rec_dir)[-1]
        )

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, window_name, None, None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0]
                + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.binocular = meta_info.get("Eye Mode", "monocular") == "binocular"
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.0
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get("min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT)
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT
        )

        # populated by producers
        g_pool.pupil_positions = pm.Bisector()
        g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()
        g_pool.eye_movements = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {"subject": "min_data_confidence_changed"}
            notification["_notify_time_"] = time() + 0.8
            g_pool.ipc_pub.notify(notification)

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop
            export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx))

            export_dir = os.path.join(g_pool.rec_dir, "exports")
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                "Player Software Version": str(g_pool.version),
                "Data Format Version": meta_info["Data Format Version"],
                "Export Date": strftime("%d.%m.%Y", localtime()),
                "Export Time": strftime("%H:%M:%S", localtime()),
                "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(),
                "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(),
                "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(),
            }
            with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
                write_key_value_file(csv, export_info)

            notification = {
                "subject": "should_export",
                "range": export_range,
                "ts_window": export_ts_window,
                "export_dir": export_dir,
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify(
                {
                    "subject": "player_process.should_start",
                    "rec_dir": rec_dir,
                    "delay": 2.0,
                }
            )

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu(
            "User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline"
        )
        g_pool.user_timelines.color = RGBA(a=0.0)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)),
                label="Interface Size",
            )
        )
        general_settings.append(
            ui.Info_Text("Player Version: {}".format(g_pool.version))
        )
        general_settings.append(
            ui.Info_Text(
                "Capture Version: {}".format(meta_info["Capture Software Version"])
            )
        )
        general_settings.append(
            ui.Info_Text(
                "Data Format Version: {}".format(meta_info["Data Format Version"])
            )
        )

        general_settings.append(
            ui.Info_Text(
                "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
            )
        )
        general_settings.append(
            ui.Slider(
                "min_data_confidence",
                g_pool,
                setter=set_data_confidence,
                step=0.05,
                min=0.0,
                max=1.0,
                label="Minimum data confidence",
            )
        )

        general_settings.append(
            ui.Button("Restart with default settings", reset_restart)
        )

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100))
        g_pool.export_button = ui.Thumb(
            "export",
            label=chr(0xE2C5),
            getter=lambda: False,
            setter=do_export,
            hotkey="e",
            label_font="pupil_icons",
        )
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [
            ("Plugin_Manager", {}),
            ("Seek_Control", {}),
            ("Log_Display", {}),
            ("Raw_Data_Exporter", {}),
            ("Vis_Polyline", {}),
            ("Vis_Circle", {}),
            ("System_Graphs", {}),
            ("System_Timelines", {}),
            ("World_Video_Exporter", {}),
            ("Pupil_From_Recording", {}),
            ("GazeFromRecording", {}),
            ("Audio_Playback", {}),
        ]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins)
        )

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                "rel_time_trim_section",
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label="Relative time range to export",
            ),
        )
        general_settings.insert(
            -1,
            ui.Text_Input(
                "frame_idx_trim_section",
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label="Frame index range to export",
            ),
        )

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n["subject"]
            if subject == "start_plugin":
                g_pool.plugins.add(
                    g_pool.plugin_by_name[n["name"]], args=n.get("args", {})
                )
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify(
                    {"subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__}
                )
                for p in g_pool.plugins:
                    if (
                        p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify
                    ):
                        ipc_pub.notify(
                            {
                                "subject": "meta.doc",
                                "actor": p.class_name,
                                "doc": p.on_notify.__doc__,
                            }
                        )

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events["pupil"] = []
            events["gaze"] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(
                        main_window, user_input.clipboard.encode()
                    )

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    # call `on_click` callbacks until a plugin indicates
                    # that it has consumed the event (by returning True)
                    any(p.on_click(pos, button, action) for p in g_pool.plugins)

                for key, scancode, action, mods in user_input.keys:
                    # call `on_key` callbacks until a plugin indicates
                    # that it has consumed the event (by returning True)
                    any(p.on_key(key, scancode, action, mods) for p in g_pool.plugins)

                for char_ in user_input.chars:
                    # call `char_` callbacks until a plugin indicates
                    # that it has consumed the event (by returning True)
                    any(p.on_char(char_) for p in g_pool.plugins)

                # present frames at appropriate speed
                g_pool.seek_control.wait(events["frame"].timestamp)
                glfw.glfwSwapBuffers(main_window)

            glfw.glfwPollEvents()

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["min_data_confidence"] = g_pool.min_data_confidence
        session_settings[
            "min_calibration_confidence"
        ] = g_pool.min_calibration_confidence
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
        session_settings["version"] = str(g_pool.version)

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Player crashed with trace:\n{}".format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "player_process.stopped"})
        sleep(1.0)
Beispiel #51
0
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
    hide_ui=False,
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
        ``set_detection_mapping_mode``: Sets detection method
        ``eye_process.should_stop``: Stops the eye process
        ``recording.started``: Starts recording eye video
        ``recording.stopped``: Stops recording eye video
        ``frame_publishing.started``: Starts frame publishing
        ``frame_publishing.stopped``: Stops frame publishing
        ``start_eye_plugin``: Start plugins in eye process

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport
        from ui_roi import UIRoi

        # monitoring
        import psutil

        # Plug-ins
        from plugin import Plugin_List

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, MPEG_Writer
        from ndsi import H264Writer
        from video_capture import source_classes, manager_classes

        from background_helper import IPC_Logging_Task_Proxy
        from pupil_detector_plugins import available_detector_plugins
        from pupil_detector_plugins.manager import PupilDetectorManager

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            # NOTE: Interrupt is handled in world/service/player which are responsible for
            # shutting down the eye process properly

        signal.signal(signal.SIGINT, interrupt_handler)

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = "capture"
        g_pool.eye_id = eye_id
        g_pool.process = f"eye{eye_id}"
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        default_detector_cls, available_detectors = available_detector_plugins(
        )
        plugins = (manager_classes + source_classes + available_detectors +
                   [PupilDetectorManager])
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        preferred_names = [
            f"Pupil Cam3 ID{eye_id}",
            f"Pupil Cam2 ID{eye_id}",
            f"Pupil Cam1 ID{eye_id}",
        ]
        if eye_id == 0:
            preferred_names += ["HD-6000"]
        default_capture_settings = (
            "UVC_Source",
            {
                "preferred_names": preferred_names,
                "frame_size": (320, 240),
                "frame_rate": 120,
            },
        )

        default_plugins = [
            # TODO: extend with plugins
            default_capture_settings,
            ("UVC_Manager", {}),
            # Detector needs to be loaded first to set `g_pool.pupil_detector`
            (default_detector_cls.__name__, {}),
            ("PupilDetectorManager", {}),
        ]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor

            active_window = glfw.glfwGetCurrentContext()
            glfw.glfwMakeContextCurrent(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x *= hdpi_factor
            y *= hdpi_factor
            g_pool.gui.update_mouse(x, y)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, g_pool.capture.frame_size)
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir,
                         "user_settings_eye{}".format(eye_id)))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.iconified = False
        g_pool.capture = None
        g_pool.flip = session_settings.get("flip", False)
        g_pool.display_mode = session_settings.get("display_mode",
                                                   "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image":
            "Raw eye camera image. This uses the least amount of CPU power",
            "roi":
            "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm":
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.glfwInit()
        if hide_ui:
            glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0)  # hide window
        title = "Pupil Capture - eye {}".format(eye_id)

        width, height = session_settings.get("window_size",
                                             (640 + icon_bar_width, 480))

        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get("window_position",
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(
            np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2],
                label="Interface Size",
            ))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width *= 2
            f_height *= 2
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        def uroi_on_mouse_button(button, action, mods):
            if g_pool.display_mode == "roi":
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    x *= hdpi_factor
                    y *= hdpi_factor
                    pos = normalize((x, y), camera_render_size)
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(
                        pos,
                        g_pool.capture.frame_size)  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,
                                                     g_pool.u_r.handle_size,
                                                     g_pool.u_r.handle_size):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            ))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])

        general_settings.append(g_pool.display_mode_info)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)
        toggle_general_settings(False)

        plugins_to_load = session_settings.get("loaded_plugins",
                                               default_plugins)
        if overwrite_cap_settings:
            # Ensure that overwrite_cap_settings takes preference over source plugins
            # with incorrect settings that were loaded from session settings.
            plugins_to_load.append(overwrite_cap_settings)

        g_pool.plugins = Plugin_List(g_pool, plugins_to_load)

        g_pool.writer = None

        g_pool.u_r = UIRoi(
            (g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get("roi")
        if roi_user_settings and tuple(
                roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        record_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        start_time_synced = notification["start_time_synced"]
                        logger.info(
                            "Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(record_path,
                                                  "eye{}.mp4".format(eye_id))
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(
                                video_path, start_time_synced)
                        elif hasattr(g_pool.capture._recent_frame,
                                     "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = MPEG_Writer(
                                video_path, start_time_synced)
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify({
                        "subject": "meta.doc",
                        "actor": "eye{}".format(eye_id),
                        "doc": eye.__doc__,
                    })
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (subject.startswith("start_eye_plugin")
                      and notification["target"] == g_pool.process):
                    try:
                        g_pool.plugins.add(
                            g_pool.plugin_by_name[notification["name"]],
                            notification.get("args", {}),
                        )
                    except KeyError as err:
                        logger.error(f"Attempt to load unknown plugin: {err}")

                for plugin in g_pool.plugins:
                    plugin.on_notify(notification)

            event = {}
            for plugin in g_pool.plugins:
                plugin.recent_events(event)

            frame = event.get("frame")
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
                        f_height,
                        f_width,
                ):
                    g_pool.pupil_detector.on_resolution_change(
                        (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
                        g_pool.capture.frame_size,
                    )
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.
                                format(type(frame), frame_publish_format))
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send({
                            "topic":
                            "frame.eye.{}".format(eye_id),
                            "width":
                            frame.width,
                            "height":
                            frame.height,
                            "index":
                            frame.index,
                            "timestamp":
                            frame.timestamp,
                            "format":
                            frame_publish_format,
                            "__raw_data__": [data],
                        })

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                result = event.get("pupil_detection_result", None)
                if result is not None:
                    pupil_socket.send(result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    glViewport(0, 0, *camera_render_size)
                    for p in g_pool.plugins:
                        p.gl_display()

                    glViewport(0, 0, *camera_render_size)
                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == "roi":
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    glViewport(0, 0, *window_size)
                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    try:
                        clipboard = glfw.glfwGetClipboardString(
                            main_window).decode()
                    except AttributeError:  # clipboard is None, might happen on startup
                        clipboard = ""
                    g_pool.gui.update_clipboard(clipboard)
                    user_input = g_pool.gui.update()
                    if user_input.clipboard != clipboard:
                        # only write to clipboard if content changed
                        glfw.glfwSetClipboardString(
                            main_window, user_input.clipboard.encode())

                    for button, action, mods in user_input.buttons:
                        x, y = glfw.glfwGetCursorPos(main_window)
                        pos = x * hdpi_factor, y * hdpi_factor
                        pos = normalize(pos, camera_render_size)
                        # Position in img pixels
                        pos = denormalize(pos, g_pool.capture.frame_size)

                        for plugin in g_pool.plugins:
                            if plugin.on_click(pos, button, action):
                                break

                    for key, scancode, action, mods in user_input.keys:
                        for plugin in g_pool.plugins:
                            if plugin.on_key(key, scancode, action, mods):
                                break

                    for char_ in user_input.chars:
                        for plugin in g_pool.plugins:
                            if plugin.on_char(char_):
                                break

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer.release()
            g_pool.writer = None

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        # save session persistent settings
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["roi"] = g_pool.u_r.get()
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)

        if not hide_ui:
            glfw.glfwRestoreWindow(
                main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.glfwGetWindowPos(
                main_window)
            session_window_size = glfw.glfwGetWindowSize(main_window)
            if 0 not in session_window_size:
                session_settings["window_size"] = session_window_size

        session_settings.close()

        for plugin in g_pool.plugins:
            plugin.alive = False
        g_pool.plugins.clean()

        glfw.glfwDestroyWindow(main_window)
        g_pool.gui.terminate()
        glfw.glfwTerminate()
        logger.info("Process shutting down.")
Beispiel #52
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version, debug):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime

    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        # display
        import glfw

        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils

        # capture
        from video_capture import File_Source

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from pupil_recording import PupilRecording
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation

        from seek_control import Seek_Control
        from surface_tracker import Surface_Tracker_Offline

        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producer.gaze_from_recording import GazeFromRecording
        from gaze_producer.gaze_from_offline_calibration import (
            GazeFromOfflineCalibration, )
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from video_export.plugins.imotions_exporter import iMotions_Exporter
        from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
        from video_export.plugins.world_video_exporter import World_Video_Exporter
        from head_pose_tracker.offline_head_pose_tracker import (
            Offline_Head_Pose_Tracker, )
        from video_capture import File_Source
        from video_overlay.plugins import Video_Overlay, Eye_Overlay

        from pupil_recording import (
            assert_valid_recording_type,
            InvalidRecordingException,
        )

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.27"), "pyglui out of date, please upgrade to newest version"

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, "plugins"))
        system_plugins = [
            Log_Display,
            Seek_Control,
            Plugin_Manager,
            System_Graphs,
            System_Timelines,
            Audio_Playback,
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Eye_Overlay,
            Video_Overlay,
            Offline_Fixation_Detector,
            Offline_Blink_Detection,
            Surface_Tracker_Offline,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            GazeFromRecording,
            GazeFromOfflineCalibration,
            World_Video_Exporter,
            iMotions_Exporter,
            Eye_Video_Exporter,
            Offline_Head_Pose_Tracker,
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for path in paths:
                try:
                    assert_valid_recording_type(path)
                    _restart_with_recording(path)
                    return
                except InvalidRecordingException as err:
                    logger.debug(str(err))

            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        def _restart_with_recording(rec_dir):
            logger.debug("Starting new session with '{}'".format(rec_dir))
            ipc_pub.notify({
                "subject": "player_drop_process.should_start",
                "rec_dir": rec_dir
            })
            glfw.glfwSetWindowShouldClose(g_pool.main_window, True)

        tick = delta_t()

        def get_dt():
            return next(tick)

        recording = PupilRecording(rec_dir)
        meta_info = recording.meta_info

        # log info about Pupil Platform and Platform in player.log
        logger.info("Application Version: {}".format(app_version))
        logger.info("System Info: {}".format(get_system_info()))
        logger.debug(f"Debug flag: {debug}")

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = SimpleNamespace()
        g_pool.app = "player"
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        video_path = recording.files().core().world().videos()[0].resolve()
        File_Source(
            g_pool,
            timing="external",
            source_path=video_path,
            buffered_decoding=True,
            fill_gaps=True,
        )

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        window_pos = session_settings.get("window_position",
                                          window_position_default)
        window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}"

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, window_name, None,
                                            None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.0
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT)
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT)

        # populated by producers
        g_pool.pupil_positions = pm.PupilDataBisector()
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()
        g_pool.eye_movements = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {"subject": "min_data_confidence_changed"}
            notification["_notify_time_"] = time() + 0.8
            g_pool.ipc_pub.notify(notification)

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop
            export_ts_window = pm.exact_window(g_pool.timestamps,
                                               (left_idx, right_idx))

            export_dir = os.path.join(g_pool.rec_dir, "exports")
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                "Player Software Version":
                str(g_pool.version),
                "Data Format Version":
                meta_info.min_player_version,
                "Export Date":
                strftime("%d.%m.%Y", localtime()),
                "Export Time":
                strftime("%H:%M:%S", localtime()),
                "Frame Index Range:":
                g_pool.seek_control.get_frame_index_trim_range_string(),
                "Relative Time Range":
                g_pool.seek_control.get_rel_time_trim_range_string(),
                "Absolute Time Range":
                g_pool.seek_control.get_abs_time_trim_range_string(),
            }
            with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
                write_key_value_file(csv, export_info)

            notification = {
                "subject": "should_export",
                "range": export_range,
                "ts_window": export_ts_window,
                "export_dir": export_dir,
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({
                "subject": "player_process.should_start",
                "rec_dir": rec_dir,
                "delay": 2.0,
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu("User Timelines",
                                                 pos=(0.0, -150.0),
                                                 size=(0.0, 0.0),
                                                 header_pos="headline")
        g_pool.user_timelines.color = RGBA(a=0.0)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2] +
                list(np.arange(1.5, 5.1, 0.5)),
                label="Interface Size",
            ))
        general_settings.append(
            ui.Info_Text(
                f"Minimum Player Version: {meta_info.min_player_version}"))
        general_settings.append(
            ui.Info_Text(f"Player Version: {g_pool.version}"))
        general_settings.append(
            ui.Info_Text(
                f"Recording Software: {meta_info.recording_software_name}"))
        general_settings.append(
            ui.Info_Text(
                f"Recording Software Version: {meta_info.recording_software_version}"
            ))

        general_settings.append(
            ui.Info_Text(
                "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
            ))
        general_settings.append(
            ui.Slider(
                "min_data_confidence",
                g_pool,
                setter=set_data_confidence,
                step=0.05,
                min=0.0,
                max=1.0,
                label="Minimum data confidence",
            ))

        general_settings.append(
            ui.Button("Restart with default settings", reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb(
            "export",
            label=chr(0xE2C5),
            getter=lambda: False,
            setter=do_export,
            hotkey="e",
            label_font="pupil_icons",
        )
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [
            ("Plugin_Manager", {}),
            ("Seek_Control", {}),
            ("Log_Display", {}),
            ("Raw_Data_Exporter", {}),
            ("Vis_Polyline", {}),
            ("Vis_Circle", {}),
            ("System_Graphs", {}),
            ("System_Timelines", {}),
            ("World_Video_Exporter", {}),
            ("Pupil_From_Recording", {}),
            ("GazeFromRecording", {}),
            ("Audio_Playback", {}),
        ]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins))

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                "rel_time_trim_section",
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label="Relative time range to export",
            ),
        )
        general_settings.insert(
            -1,
            ui.Text_Input(
                "frame_idx_trim_section",
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label="Frame index range to export",
            ),
        )

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n["subject"]
            if subject == "start_plugin":
                g_pool.plugins.add(g_pool.plugin_by_name[n["name"]],
                                   args=n.get("args", {}))
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify({
                    "subject": "meta.doc",
                    "actor": g_pool.app,
                    "doc": player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            "subject": "meta.doc",
                            "actor": p.class_name,
                            "doc": p.on_notify.__doc__,
                        })

        while (not glfw.glfwWindowShouldClose(main_window)
               and not process_was_interrupted):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events["pupil"] = []
            events["gaze"] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            glfw.glfwPollEvents()
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                # present frames at appropriate speed
                g_pool.seek_control.wait(events["frame"].timestamp)
                glfw.glfwSwapBuffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["min_data_confidence"] = g_pool.min_data_confidence
        session_settings[
            "min_calibration_confidence"] = g_pool.min_calibration_confidence
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["window_position"] = glfw.glfwGetWindowPos(
            main_window)
        session_settings["version"] = str(g_pool.version)

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Player crashed with trace:\n{}".format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "player_process.stopped"})
        sleep(1.0)
Beispiel #53
0
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """
    is_alive = Is_Alive_Manager(is_alive_flag)
    with is_alive:
        import logging
        # Set up root logger for this process before doing imports of logged modules.
        logger = logging.getLogger()
        logger.setLevel(logging.INFO)
        # remove inherited handlers
        logger.handlers = []
        # create file handler which logs even debug messages
        fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w')
        # fh.setLevel(logging.DEBUG)
        # create console handler with a higher log level
        ch = logging.StreamHandler()
        ch.setLevel(logger.level+10)
        # create formatter and add it to the handlers
        formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
        ch.setFormatter(formatter)
        # add the handlers to the logger
        logger.addHandler(fh)
        logger.addHandler(ch)
        #silence noisy modules
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        # create logger for the context of this function
        logger = logging.getLogger(__name__)


        # We deferr the imports becasue of multiprocessing.
        # Otherwise the world process each process also loads the other imports.

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui,graph,cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
        import OpenGL.GL as gl
        from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based
        from ui_roi import UIRoi
        #monitoring
        import psutil
        import math


        # helpers/utils
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
        from av_writer import JPEG_Writer,AV_Writer

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}



        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600,300*eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600,31+300*eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600,300*eye_id)


        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.pupil_queue = pupil_queue
        g_pool.timebase = timebase


        # Callback functions
        def on_resize(window,w, h):
            if not g_pool.iconified:
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w,h)
                graph.adjust_size(w,h)
                adjust_gl_view(w,h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key,scancode,action,mods)

        def on_char(window,char):
            g_pool.gui.update_char(char)

        def on_iconify(window,iconified):
            g_pool.iconified = iconified

        def on_button(window,button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1-pos[0],1-pos[1]
                    pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
                        return # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button,action,mods)



        def on_pos(window,x, y):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height) )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)

        def on_scroll(window,x,y):
            g_pool.gui.update_scroll(x,y*scroll_factor)


        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
        if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
            logger.info("Session setting are from older version of this app. I will not use those.")
            session_settings.clear()
        # Initialize capture
        cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
        default_settings = {'frame_size':(640,480),'frame_rate':60}
        previous_settings = session_settings.get('capture_settings',None)
        if previous_settings and previous_settings['name'] == cap.name:
            cap.settings = previous_settings
        else:
            cap.settings = default_settings


        # Test capture
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Could not retrieve image from capture")
            cap.close()
            return

        #signal world that we are ready to go
        # pipe_to_world.send('eye%s process ready'%eye_id)

        # any object we attach to the g_pool object *from now on* will only be visible to this process!
        # vars should be declared here to make them visible to the code reader.
        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.flip = session_settings.get('flip',False)
        g_pool.display_mode = session_settings.get('display_mode','camera_image')
        g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                    'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
                                    'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}


        g_pool.u_r = UIRoi(frame.img.shape)
        g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get()))


        def on_frame_size_change(new_size):
            g_pool.u_r = UIRoi((new_size[1],new_size[0]))

        cap.on_frame_size_change = on_frame_size_change

        writer = None

        pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
        last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()


        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]


        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)


        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s"%eye_id
        width,height = session_settings.get('window_size',(frame.width, frame.height))
        main_window = glfw.glfwCreateWindow(width,height, title, None, None)
        window_pos = session_settings.get('window_position',window_position_default)
        glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_frame(frame)
        glfw.glfwSwapInterval(0)

        sphere  = Sphere(20)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale',1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
        general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
        general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
        general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
        g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.sidebar.append(general_settings)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
        general_settings.append(detector_selector)

        # let detector add its GUI
        g_pool.pupil_detector.init_gui(g_pool.sidebar)
        # let the camera add its GUI
        g_pool.capture.init_gui(g_pool.sidebar)


        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
        glfw.glfwSetKeyCallback(main_window,on_key)
        glfw.glfwSetCharCallback(main_window,on_char)
        glfw.glfwSetMouseButtonCallback(main_window,on_button)
        glfw.glfwSetCursorPosCallback(main_window,on_pos)
        glfw.glfwSetScrollCallback(main_window,on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))


        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config',{})


        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = frame.timestamp

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20,130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140,130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"


        #create a timer to control window update frequency
        window_update_timer = timer(1/60.)
        def window_should_update():
            return next(window_update_timer)


        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if pipe_to_world.poll():
                cmd = pipe_to_world.recv()
                if cmd == 'Exit':
                    break
                elif cmd == "Ping":
                    pipe_to_world.send("Pong")
                    command = None
                else:
                    command,payload = cmd
                if command == 'Set_Detection_Mapping_Mode':
                    if payload == '3d':
                        if not isinstance(g_pool.pupil_detector,Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only  = True
                    else:
                        set_detector(Detector_2D)
                        detector_selector.read_only = False

            else:
                command = None



            # Get an image from the grabber
            try:
                frame = cap.get_frame()
            except CameraCaptureError:
                logger.error("Capture from Camera Failed. Stopping.")
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                cap.seek_to_frame(0)
                frame = cap.get_frame()


            #update performace graphs
            t = frame.timestamp
            dt,ts = t-ts,t
            try:
                fps_graph.add(1./dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()


            ###  RECORDING of Eye Video (on demand) ###
            # Setup variables and lists for recording
            if 'Rec_Start' == command:
                record_path,raw_mode = payload
                logger.info("Will save eye video to: %s"%record_path)
                timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                    writer = JPEG_Writer(video_path,cap.frame_rate)
                else:
                    video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                    writer = AV_Writer(video_path,cap.frame_rate)
                timestamps = []
            elif 'Rec_Stop' == command:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path,np.asarray(timestamps))
                del timestamps

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)


            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            g_pool.pupil_queue.put(result)

            # GL drawing
            if window_should_update():
                if not g_pool.iconified:
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image','roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    window_size =  glfw.glfwGetWindowSize(main_window)
                    make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)

                    if result['method'] == '3d c++':

                        eye_ball = result['projected_sphere']
                        try:
                            pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
                                                (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
                                                int(eye_ball['angle']),0,360,8)
                        except ValueError as e:
                            pass
                        else:
                            draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )

                    if result['confidence'] >0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
                                            (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
                                            int(result['ellipse']['angle']),0,360,15)
                            confidence = result['confidence'] * 0.7 #scale it a little
                            draw_polyline(pts,1,RGBA(1.,0,0,confidence))
                            draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize() #detector decides if we visualize or not


        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path,np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window) #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = g_pool.version
        session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        cap.close()


        logger.debug("Process done")
Beispiel #54
0
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url,
        user_dir, version, eye_id, overwrite_cap_settings=None):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools
    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):

        # logging setup
        import logging
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logger = logging.getLogger()
        logger.handlers = []
        logger.setLevel(logging.INFO)
        logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        # general imports
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible
        from ui_roi import UIRoi
        # monitoring
        import psutil

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, AV_Writer
        from ndsi import H264Writer
        from video_capture import source_classes
        from video_capture import manager_classes

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {Detector_2D.__name__: Detector_2D,
                           Detector_3D.__name__: Detector_3D}

        # UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 10.0
            window_position_default = (600,31+ 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        # g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.process = 'eye{}'.format(eye_id)
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value
        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window, w, h):
            if is_window_visible(window):
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
                g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
                g_pool.gui.update_window(w, h)
                g_pool.gui.collect_menus()
                for g in g_pool.graphs:
                    g.scale = hdpi_factor
                    g.adjust_window_size(w, h)
                adjust_gl_view(w, h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            hdpi_factor = glfw.glfwGetFramebufferSize(
                window)[0] / glfw.glfwGetWindowSize(window)[0]
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,g_pool.capture.frame_size )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode('utf-8') for x in range(count)]
            g_pool.capture_manager.on_drop(paths)
            g_pool.capture.on_drop(paths)

        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id)))
        if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
            logger.info("Session setting are from a different version of this app. I will not use those.")
            session_settings.clear()


        g_pool.iconified = False
        g_pool.capture = None
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get('flip', False)
        g_pool.display_mode = session_settings.get(
            'display_mode', 'camera_image')
        g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                         'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
                                         'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}


        capture_manager_settings = session_settings.get(
            'capture_manager_settings', ('UVC_Manager',{}))

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__:c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)


        if eye_id == 0:
            cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
        else:
            cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]

        # Initialize capture
        default_settings = ('UVC_Source',{
                            'preferred_names'  : cap_src,
                            'frame_size': (640,480),
                            'frame_rate': 90
                            })

        capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
        source_class_name, source_settings = capture_source_settings
        source_class_by_name = {c.__name__:c for c in source_classes}
        g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
        assert g_pool.capture

        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get('roi')
        if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        pupil_detector_settings = session_settings.get(
            'pupil_detector_settings', None)
        last_pupil_detector = pupil_detectors[session_settings.get(
            'last_pupil_detector', Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(
            g_pool, pupil_detector_settings)

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)

        # Initialize glfw
        glfw.glfwInit()
        title = "Pupil Capture - eye {}".format(eye_id)
        width, height = session_settings.get(
            'window_size', g_pool.capture.frame_size)
        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get(
            'window_position', window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                           pos=(-300, 0),
                                           size=(0, 0),
                                           header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(ui.Selector('gui_user_scale', g_pool,
                                          setter=set_scale,
                                          selection=[.8, .9, 1., 1.1, 1.2],
                                          label='Interface Size'))
        general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) )
        general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
        general_settings.append(ui.Selector('display_mode',
                                            g_pool,
                                            setter=set_display_mode_info,
                                            selection=['camera_image','roi','algorithm'],
                                            labels=['Camera Image', 'ROI', 'Algorithm'],
                                            label="Mode")
                                            )
        g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])

        general_settings.append(g_pool.display_mode_info)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector('pupil_detector',
                                        getter=lambda: g_pool.pupil_detector.__class__,
                                        setter=set_detector, selection=[
                                            Detector_2D, Detector_3D],
                                        labels=['C++ 2d detector',
                                                'C++ 3d detector'],
                                        label="Detection method")
        general_settings.append(detector_selector)

        g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
        g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
        g_pool.capture_source_menu.collapsed = True
        g_pool.capture.init_gui()

        g_pool.sidebar.append(general_settings)
        g_pool.sidebar.append(g_pool.capture_selector_menu)
        g_pool.sidebar.append(g_pool.capture_source_menu)

        g_pool.pupil_detector.init_gui(g_pool.sidebar)

        g_pool.capture_manager.init_gui()
        g_pool.writer = None

        def replace_source(source_class_name,source_settings):
            g_pool.capture.cleanup()
            g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
            g_pool.capture.init_gui()
            if g_pool.writer:
                logger.info("Done recording.")
                g_pool.writer.release()
                g_pool.writer = None

        g_pool.replace_source = replace_source # for ndsi capture

        def replace_manager(manager_class):
            g_pool.capture_manager.cleanup()
            g_pool.capture_manager = manager_class(g_pool)
            g_pool.capture_manager.init_gui()

        #We add the capture selection menu, after a manager has been added:
        g_pool.capture_selector_menu.insert(0,ui.Selector(
                                                'capture_manager',g_pool,
                                                setter    = replace_manager,
                                                getter    = lambda: g_pool.capture_manager.__class__,
                                                selection = manager_classes,
                                                labels    = [b.gui_name for b in manager_classes],
                                                label     = 'Manager'
                                            ))


        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = 'jpeg'

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning('Process started.')

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification['subject']
                if subject.startswith('eye_process.should_stop'):
                    if notification['eye_id'] == eye_id:
                        break
                elif subject == 'set_detection_mapping_mode':
                    if notification['mode'] == '3d':
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    else:
                        if not isinstance(g_pool.pupil_detector, Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                elif subject == 'recording.started':
                    if notification['record_eye'] and g_pool.capture.online:
                        record_path = notification['rec_path']
                        raw_mode = notification['compression']
                        logger.info("Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id))
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate)
                        elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'):
                            g_pool.writer = H264Writer(video_path,
                                                       g_pool.capture.frame_size[0],
                                                       g_pool.capture.frame_size[1],
                                                       g_pool.capture.frame_rate)
                        else:
                            g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate)
                elif subject == 'recording.stopped':
                    if g_pool.writer:
                        logger.info("Done recording.")
                        g_pool.writer.release()
                        g_pool.writer = None
                elif subject.startswith('meta.should_doc'):
                    ipc_socket.notify({
                        'subject': 'meta.doc',
                        'actor': 'eye{}'.format(eye_id),
                        'doc': eye.__doc__
                    })
                elif subject.startswith('frame_publishing.started'):
                    should_publish_frames = True
                    frame_publish_format = notification.get('format', 'jpeg')
                elif subject.startswith('frame_publishing.stopped'):
                    should_publish_frames = False
                    frame_publish_format = 'jpeg'
                elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process:
                    replace_source(notification['name'],notification['args'])

                g_pool.capture.on_notify(notification)

            # Get an image from the grabber
            event = {}
            g_pool.capture.recent_events(event)
            frame = event.get('frame')
            g_pool.capture_manager.recent_events(event)
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width):
                    g_pool.pupil_detector.on_resolution_change((g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size)
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        else:
                            raise AttributeError()
                    except AttributeError:
                        pass
                    else:
                        pupil_socket.send('frame.eye.%s'%eye_id,{
                            'width': frame.width,
                            'height': frame.height,
                            'index': frame.index,
                            'timestamp': frame.timestamp,
                            'format': frame_publish_format,
                            '__raw_data__': [data]
                        })

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1./dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                # pupil ellipse detection
                result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
                result['id'] = eye_id

                # stream the result
                pupil_socket.send('pupil.%s'%eye_id,result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    if frame:
                        # switch to work in normalized coordinate space
                        if g_pool.display_mode == 'algorithm':
                            g_pool.image_tex.update_from_ndarray(frame.img)
                        elif g_pool.display_mode in ('camera_image', 'roi'):
                            g_pool.image_tex.update_from_ndarray(frame.gray)
                        else:
                            pass
                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()
                    f_width, f_height = g_pool.capture.frame_size
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    if frame:
                        if result['method'] == '3d c++':
                            eye_ball = result['projected_sphere']
                            try:
                                pts = cv2.ellipse2Poly(
                                    (int(eye_ball['center'][0]),
                                     int(eye_ball['center'][1])),
                                    (int(eye_ball['axes'][0] / 2),
                                     int(eye_ball['axes'][1] / 2)),
                                    int(eye_ball['angle']), 0, 360, 8)
                            except ValueError as e:
                                pass
                            else:
                                draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence']))
                        if result['confidence'] > 0:
                            if 'ellipse' in result:
                                pts = cv2.ellipse2Poly(
                                    (int(result['ellipse']['center'][0]),
                                     int(result['ellipse']['center'][1])),
                                    (int(result['ellipse']['axes'][0] / 2),
                                     int(result['ellipse']['axes'][1] / 2)),
                                    int(result['ellipse']['angle']), 0, 360, 15)
                                confidence = result['confidence'] * 0.7
                                draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
                                draw_points([result['ellipse']['center']],
                                            size=20,
                                            color=RGBA(1., 0., 0., confidence),
                                            sharpness=1.)

                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    g_pool.gui.update()

                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize()  # detector decides if we visualize or not

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer = None

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict()
        session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.capture.deinit_gui()
        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()
        logger.info("Process shutting down.")
Beispiel #55
0
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url,
          ipc_push_url, user_dir, version):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    # display
    import glfw
    from pyglui import ui, graph, cygl, __version__ as pyglui_version
    assert pyglui_version >= '1.7'
    from pyglui.cygl.utils import Named_Texture
    import gl_utils

    # monitoring
    import psutil

    # helpers/utils
    from version_utils import VersionFormat
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t, get_system_info, timer
    from uvc import get_time_monotonic
    logger.info('Application Version: {}'.format(version))
    logger.info('System Info: {}'.format(get_system_info()))

    import audio

    # trigger pupil detector cpp build:
    import pupil_detectors
    del pupil_detectors

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin
    from fixation_detector import Fixation_Detector
    from recorder import Recorder
    from display_recent_gaze import Display_Recent_Gaze
    from time_sync import Time_Sync
    from pupil_remote import Pupil_Remote
    from pupil_groups import Pupil_Groups
    from surface_tracker import Surface_Tracker
    from log_display import Log_Display
    from annotations import Annotation_Capture
    from log_history import Log_History
    from frame_publisher import Frame_Publisher
    from blink_detection import Blink_Detection
    from video_capture import source_classes, manager_classes, Base_Manager
    from pupil_data_relay import Pupil_Data_Relay
    from remote_recorder import Remote_Recorder
    from audio_capture import Audio_Capture
    from accuracy_visualizer import Accuracy_Visualizer

    # UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (0, 0)
    elif platform.system() == 'Windows':
        scroll_factor = 10.0
        window_position_default = (8, 31)
    else:
        scroll_factor = 1.0
        window_position_default = (0, 0)

    # g_pool holds variables for this process they are accesible to all plugins
    g_pool = Global_Container()
    g_pool.app = 'capture'
    g_pool.process = 'world'
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.timebase = timebase
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url
    g_pool.eyes_are_alive = eyes_are_alive

    def get_timestamp():
        return get_time_monotonic() - g_pool.timebase.value
    g_pool.get_timestamp = get_timestamp
    g_pool.get_now = get_time_monotonic

    # manage plugins
    runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins'))
    calibration_plugins += [p for p in runtime_plugins if issubclass(p, Calibration_Plugin)]
    runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Calibration_Plugin)]
    manager_classes += [p for p in runtime_plugins if issubclass(p, Base_Manager)]
    runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Base_Manager)]
    user_launchable_plugins = [Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker,
                               Annotation_Capture, Log_History, Fixation_Detector,
                               Blink_Detection, Remote_Recorder, Accuracy_Visualizer] + runtime_plugins
    system_plugins = [Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay]
    plugin_by_index = (system_plugins + user_launchable_plugins + calibration_plugins
                       + gaze_mapping_plugins + manager_classes + source_classes)
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index, plugin_by_index))

    default_capture_settings = {
        'preferred_names': ["Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)",
                            "C510", "B525", "C525", "C615", "C920", "C930e"],
        'frame_size': (1280, 720),
        'frame_rate': 30
    }

    default_plugins = [("UVC_Source", default_capture_settings),
                       ('Pupil_Data_Relay', {}),
                       ('UVC_Manager', {}),
                       ('Log_Display', {}),
                       ('Dummy_Gaze_Mapper', {}),
                       ('Display_Recent_Gaze', {}),
                       ('Screen_Marker_Calibration', {}),
                       ('Recorder', {}),
                       ('Pupil_Remote', {})]

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_window_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_window_char(window, char):
        g_pool.gui.update_char(char)

    def on_window_mouse_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(
            window)[0] / glfw.glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    def on_drop(window, count, paths):
        paths = [paths[x].decode('utf-8') for x in range(count)]
        for p in g_pool.plugins:
            p.on_drop(paths)

    tick = delta_t()

    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_world'))
    if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
        logger.info("Session setting are from a different version of this app. I will not use those.")
        session_settings.clear()

    g_pool.iconified = False
    g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '3d')
    g_pool.active_calibration_plugin = None
    g_pool.active_gaze_mapping_plugin = None
    g_pool.capture_manager = None

    audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def launch_eye_process(eye_id, delay=0):
        n = {'subject': 'eye_process.should_start.{}'.format(eye_id),
             'eye_id': eye_id, 'delay': delay}
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id,'delay':0.2}
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
        ipc_pub.notify(n)

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'set_detection_mapping_mode':
            if n['mode'] == '2d':
                if ("Vector_Gaze_Mapper" in
                        g_pool.active_gaze_mapping_plugin.class_name):
                    logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
                    g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
            g_pool.detection_mapping_mode = n['mode']
        elif subject == 'start_plugin':
            g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {}))
        elif subject == 'stop_plugin':
            for p in g_pool.plugins:
                if p.class_name == n['name']:
                    p.alive = False
                    g_pool.plugins.clean()
        elif subject == 'eye_process.started':
            n = {'subject': 'set_detection_mapping_mode',
                 'mode': g_pool.detection_mapping_mode}
            ipc_pub.notify(n)
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({'subject': 'meta.doc',
                            'actor': g_pool.app,
                            'doc': world.__doc__})
            for p in g_pool.plugins:
                if (p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify):
                    ipc_pub.notify({'subject': 'meta.doc',
                                    'actor': p.class_name,
                                    'doc': p.on_notify.__doc__})

    # window and gl setup
    glfw.glfwInit()
    width, height = session_settings.get('window_size', (1280, 720))
    main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World")
    window_pos = session_settings.get('window_position', window_position_default)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    def reset_restart():
        logger.warning("Resetting all settings and restarting Capture.")
        glfw.glfwSetWindowShouldClose(main_window, True)
        ipc_pub.notify({'subject': 'reset_restart_process.should_start'})

    # setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Button('Reset to default settings',reset_restart))
    general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface size'))
    general_settings.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window,g_pool.capture.frame_size[0],g_pool.capture.frame_size[1])) )
    general_settings.append(ui.Selector('audio_mode', audio, selection=audio.audio_modes))
    general_settings.append(ui.Selector('detection_mapping_mode',
                                        g_pool,
                                        label='detection & mapping mode',
                                        setter=set_detection_mapping_mode,
                                        selection=['2d','3d']
                                    ))
    general_settings.append(ui.Switch('eye0_process',
                                        label='Detect eye 0',
                                        setter=lambda alive: start_stop_eye(0,alive),
                                        getter=lambda: eyes_are_alive[0].value
                                    ))
    general_settings.append(ui.Switch('eye1_process',
                                        label='Detect eye 1',
                                        setter=lambda alive: start_stop_eye(1,alive),
                                        getter=lambda: eyes_are_alive[1].value
                                    ))
    selector_label = "Select to load"
    labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins]
    user_launchable_plugins.insert(0, selector_label)
    labels.insert(0, selector_label)
    general_settings.append(ui.Selector('Open plugin',
                                        selection=user_launchable_plugins,
                                        labels=labels,
                                        setter=open_plugin,
                                        getter=lambda: selector_label))

    general_settings.append(ui.Info_Text('Capture Version: {}'.format(g_pool.version)))

    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))

    g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
    g_pool.capture_source_menu.collapsed = True

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.calibration_menu.collapsed = True
    g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')

    g_pool.sidebar.append(general_settings)
    g_pool.sidebar.append(g_pool.capture_selector_menu)
    g_pool.sidebar.append(g_pool.capture_source_menu)
    g_pool.sidebar.append(g_pool.calibration_menu)

    g_pool.gui.append(g_pool.sidebar)
    g_pool.gui.append(g_pool.quickbar)

    # plugins that are loaded based on user settings from previous session
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(0,ui.Selector(
                        'active_calibration_plugin',
                        getter=lambda: g_pool.active_calibration_plugin.__class__,
                        selection = calibration_plugins,
                        labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
                        setter= open_plugin,label='Method'
                                ))

    #We add the capture selection menu, after a manager has been added:
    g_pool.capture_selector_menu.insert(0,ui.Selector(
                            'capture_manager',
                            setter    = open_plugin,
                            getter    = lambda: g_pool.capture_manager.__class__,
                            selection = manager_classes,
                            labels    = [b.gui_name for b in manager_classes],
                            label     = 'Manager'
                        ))

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfw.glfwSetKeyCallback(main_window, on_window_key)
    glfw.glfwSetCharCallback(main_window, on_window_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)
    glfw.glfwSetDropCallback(main_window, on_drop)

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # create a timer to control window update frequency
    window_update_timer = timer(1 / 60)
    def window_should_update():
        return next(window_update_timer)

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = g_pool.get_timestamp()

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil0_graph = graph.Bar_Graph(max_val=1.0)
    pupil0_graph.pos = (260, 130)
    pupil0_graph.update_rate = 5
    pupil0_graph.label = "id0 conf: %0.2f"
    pupil1_graph = graph.Bar_Graph(max_val=1.0)
    pupil1_graph.pos = (380, 130)
    pupil1_graph.update_rate = 5
    pupil1_graph.label = "id1 conf: %0.2f"
    pupil_graphs = pupil0_graph, pupil1_graph
    g_pool.graphs = [cpu_graph, fps_graph, pupil0_graph, pupil1_graph]

    # trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    if session_settings.get('eye1_process_alive', False):
        launch_eye_process(1, delay=0.6)
    if session_settings.get('eye0_process_alive', True):
        launch_eye_process(0, delay=0.3)

    ipc_pub.notify({'subject': 'world_process.started'})
    logger.warning('Process started.')

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            for p in g_pool.plugins:
                p.on_notify(n)


        #a dictionary that allows plugins to post and read events
        events = {}
        # report time between now and the last loop interation
        events['dt'] = get_dt()

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # update performace graphs
        if 'frame' in events:
            t = events["frame"].timestamp
            dt, ts = t-ts, t
            try:
                fps_graph.add(1./dt)
            except ZeroDivisionError:
                pass
        for p in events["pupil_positions"]:
            pupil_graphs[p['id']].add(p['confidence'])
        cpu_graph.update()

        # send new events to ipc:
        del events['pupil_positions']  # already on the wire
        del events['gaze_positions']  # sent earlier
        if 'frame' in events:
            del events['frame']  # send explicity with frame publisher
        if 'depth_frame' in events:
            del events['depth_frame']
        if 'audio_packets' in events:
            del events['audio_packets']
        del events['dt']  # no need to send this
        for topic, data in events.items():
            assert(isinstance(data, (list, tuple)))
            for d in data:
                ipc_pub.send(topic, d)

        glfw.glfwMakeContextCurrent(main_window)
        # render visual feedback from loaded plugins
        if window_should_update() and gl_utils.is_window_visible(main_window):
            for p in g_pool.plugins:
                p.gl_display()

            for g in g_pool.graphs:
                g.draw()

            unused_elements = g_pool.gui.update()
            for button, action, mods in unused_elements.buttons:
                pos = glfw.glfwGetCursorPos(main_window)
                pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                # Position in img pixels
                pos = denormalize(pos, g_pool.capture.frame_size)
                for p in g_pool.plugins:
                    p.on_click(pos, button, action)

            for key, scancode, action, mods in unused_elements.keys:
                for p in g_pool.plugins:
                    p.on_key(key, scancode, action, mods)

            for char_ in unused_elements.chars:
                for p in g_pool.plugins:
                    p.on_char(char_)

            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings['eye0_process_alive'] = eyes_are_alive[0].value
    session_settings['eye1_process_alive'] = eyes_are_alive[1].value
    session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
    session_settings['audio_mode'] = audio.audio_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()


    # shut down eye processes:
    stop_eye_process(0)
    stop_eye_process(1)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'world_process.stopped'})
Beispiel #56
0
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version):
    # general imports
    import logging

    # networking
    import zmq
    import zmq_tools
    from time import sleep

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        import glfw
        import gl_utils
        from OpenGL.GL import glClearColor
        from version_utils import VersionFormat
        from file_methods import Persistent_Dict
        from pyglui.pyfontstash import fontstash
        from pyglui.ui import get_roboto_font_path
        import player_methods as pm
        import update_methods as um

        def on_drop(window, count, paths):
            nonlocal rec_dir
            rec_dir = paths[0].decode("utf-8")

        if rec_dir:
            if not pm.is_pupil_rec_dir(rec_dir):
                rec_dir = None
        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player")
        )
        if VersionFormat(session_settings.get("version", "0.0")) != app_version:
            logger.info(
                "Session setting are from a  different version of this app. I will not use those."
            )
            session_settings.clear()
        w, h = session_settings.get("window_size", (1280, 720))
        window_pos = session_settings.get("window_position", window_position_default)

        glfw.glfwInit()
        glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0)
        window = glfw.glfwCreateWindow(w, h, "Pupil Player")
        glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1)

        glfw.glfwMakeContextCurrent(window)
        glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1])
        glfw.glfwSetDropCallback(window, on_drop)

        glfont = fontstash.Context()
        glfont.add_font("roboto", get_roboto_font_path())
        glfont.set_align_string(v_align="center", h_align="middle")
        glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
        gl_utils.basic_gl_setup()
        glClearColor(0.5, 0.5, 0.5, 0.0)
        text = "Drop a recording directory onto this window."
        tip = "(Tip: You can drop a recording directory onto the app icon.)"
        # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
        while not glfw.glfwWindowShouldClose(window):

            fb_size = glfw.glfwGetFramebufferSize(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            gl_utils.adjust_gl_view(*fb_size)

            if rec_dir:
                if pm.is_pupil_rec_dir(rec_dir):
                    logger.info("Starting new session with '{}'".format(rec_dir))
                    text = "Updating recording format."
                    tip = "This may take a while!"
                else:
                    logger.error("'{}' is not a valid pupil recording".format(rec_dir))
                    tip = "Oops! That was not a valid recording."
                    rec_dir = None

            gl_utils.clear_gl_screen()
            glfont.set_blur(10.5)
            glfont.set_color_float((0.0, 0.0, 0.0, 1.0))
            glfont.set_size(w / 25.0 * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, 0.3 * h * hdpi_factor, text)
            glfont.set_size(w / 30.0 * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, 0.4 * h * hdpi_factor, tip)
            glfont.set_blur(0.96)
            glfont.set_color_float((1.0, 1.0, 1.0, 1.0))
            glfont.set_size(w / 25.0 * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, 0.3 * h * hdpi_factor, text)
            glfont.set_size(w / 30.0 * hdpi_factor)
            glfont.draw_text(w / 2 * hdpi_factor, 0.4 * h * hdpi_factor, tip)

            glfw.glfwSwapBuffers(window)

            if rec_dir:
                try:
                    um.update_recording_to_recent(rec_dir)
                except AssertionError as err:
                    logger.error(str(err))
                    rec_dir = None
                else:
                    glfw.glfwSetWindowShouldClose(window, True)

            glfw.glfwPollEvents()

        session_settings["window_position"] = glfw.glfwGetWindowPos(window)
        session_settings.close()
        glfw.glfwDestroyWindow(window)
        if rec_dir:
            ipc_pub.notify(
                {"subject": "player_process.should_start", "rec_dir": rec_dir}
            )

    except:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process player_drop crashed with trace:\n{}".format(trace))

    finally:
        sleep(1.0)
Beispiel #57
0
def world(
    timebase,
    eye_procs_alive,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    preferred_remote_port,
    hide_ui,
):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            "subject": "eye_process.should_start.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": delay,
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            "subject": "eye_process.should_stop.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": 0.2,
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {"subject": "set_detection_mapping_mode", "mode": new_mode}
        ipc_pub.notify(n)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # display
        import glfw
        from version_utils import VersionFormat
        from pyglui import ui, cygl, __version__ as pyglui_version

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.24"), "pyglui out of date, please upgrade to newest version"
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic

        logger.info("Application Version: {}".format(version))
        logger.info("System Info: {}".format(get_system_info()))

        import audio

        # trigger pupil detector cpp build:
        import pupil_detectors

        del pupil_detectors

        # Plug-ins
        from plugin import (
            Plugin,
            System_Plugin_Base,
            Plugin_List,
            import_runtime_plugins,
        )
        from plugin_manager import Plugin_Manager
        from calibration_routines import (
            calibration_plugins,
            gaze_mapping_plugins,
            Calibration_Plugin,
            Gaze_Mapping_Plugin,
        )
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from pupil_remote import Pupil_Remote
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker_Online
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from frame_publisher import Frame_Publisher
        from blink_detection import Blink_Detection
        from video_capture import (
            source_classes,
            manager_classes,
            Base_Manager,
            Base_Source,
        )
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from audio_capture import Audio_Capture
        from accuracy_visualizer import Accuracy_Visualizer

        # from saccade_detector import Saccade_Detector
        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay
        from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (8, 90)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process they are accessible to all plugins
        g_pool = SimpleNamespace()
        g_pool.app = "capture"
        g_pool.process = "world"
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eye_procs_alive = eye_procs_alive
        g_pool.preferred_remote_port = preferred_remote_port

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, "plugins"))
        user_plugins = [
            Audio_Capture,
            Pupil_Groups,
            Frame_Publisher,
            Pupil_Remote,
            Time_Sync,
            Surface_Tracker_Online,
            Annotation_Capture,
            Log_History,
            Fixation_Detector,
            Blink_Detection,
            Remote_Recorder,
            Accuracy_Visualizer,
            Camera_Intrinsics_Estimation,
            Hololens_Relay,
            Online_Head_Pose_Tracker,
        ]

        system_plugins = ([
            Log_Display,
            Display_Recent_Gaze,
            Recorder,
            Pupil_Data_Relay,
            Plugin_Manager,
            System_Graphs,
        ] + manager_classes + source_classes)
        plugins = (system_plugins + user_plugins + runtime_plugins +
                   calibration_plugins + gaze_mapping_plugins)
        user_plugins += [
            p for p in runtime_plugins if not isinstance(
                p,
                (
                    Base_Manager,
                    Base_Source,
                    System_Plugin_Base,
                    Calibration_Plugin,
                    Gaze_Mapping_Plugin,
                ),
            )
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_settings = {
            "preferred_names": [
                "Pupil Cam1 ID2",
                "Logitech Camera",
                "(046d:081d)",
                "C510",
                "B525",
                "C525",
                "C615",
                "C920",
                "C930e",
            ],
            "frame_size": (1280, 720),
            "frame_rate":
            30,
        }

        default_plugins = [
            ("UVC_Source", default_capture_settings),
            ("Pupil_Data_Relay", {}),
            ("UVC_Manager", {}),
            ("Log_Display", {}),
            ("Dummy_Gaze_Mapper", {}),
            ("Display_Recent_Gaze", {}),
            ("Screen_Marker_Calibration", {}),
            ("Recorder", {}),
            ("Pupil_Remote", {}),
            ("Accuracy_Visualizer", {}),
            ("Plugin_Manager", {}),
            ("System_Graphs", {}),
        ]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()

            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_world"))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", 0.8)
        g_pool.detection_mapping_mode = session_settings.get(
            "detection_mapping_mode", "3d")
        g_pool.active_calibration_plugin = None
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.audio_mode = session_settings.get("audio_mode",
                                                audio.default_audio_mode)

        def handle_notifications(noti):
            subject = noti["subject"]
            if subject == "set_detection_mapping_mode":
                if noti["mode"] == "2d":
                    if ("Vector_Gaze_Mapper"
                            in g_pool.active_gaze_mapping_plugin.class_name):
                        logger.warning(
                            "The gaze mapper is not supported in 2d mode. Please recalibrate."
                        )
                        g_pool.plugins.add(
                            g_pool.plugin_by_name["Dummy_Gaze_Mapper"])
                g_pool.detection_mapping_mode = noti["mode"]
            elif subject == "start_plugin":
                try:
                    g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]],
                                       args=noti.get("args", {}))
                except KeyError as err:
                    logger.error(f"Attempt to load unknown plugin: {err}")
            elif subject == "stop_plugin":
                for p in g_pool.plugins:
                    if p.class_name == noti["name"]:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == "eye_process.started":
                noti = {
                    "subject": "set_detection_mapping_mode",
                    "mode": g_pool.detection_mapping_mode,
                }
                ipc_pub.notify(noti)
            elif subject == "set_min_calibration_confidence":
                g_pool.min_calibration_confidence = noti["value"]
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify({
                    "subject": "meta.doc",
                    "actor": g_pool.app,
                    "doc": world.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            "subject": "meta.doc",
                            "actor": p.class_name,
                            "doc": p.on_notify.__doc__,
                        })
            elif subject == "world_process.adapt_window_size":
                set_window_size()

        width, height = session_settings.get("window_size",
                                             (1280 + icon_bar_width, 720))

        # window and gl setup
        glfw.glfwInit()
        if hide_ui:
            glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0)  # hide window
        main_window = glfw.glfwCreateWindow(width, height,
                                            "Pupil Capture - World")
        window_pos = session_settings.get("window_position",
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({
                "subject": "world_process.should_start",
                "delay": 2.0
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is opened, the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-400, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
                                             (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.6, 0.8, 1.0, 1.2, 1.4],
                label="Interface size",
            ))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)
            on_resize(main_window, f_width, f_height)

        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Selector("audio_mode", audio, selection=audio.audio_modes))
        general_settings.append(
            ui.Selector(
                "detection_mapping_mode",
                g_pool,
                label="detection & mapping mode",
                setter=set_detection_mapping_mode,
                selection=["disabled", "2d", "3d"],
            ))
        general_settings.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: start_stop_eye(0, alive),
                getter=lambda: eye_procs_alive[0].value,
            ))
        general_settings.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: start_stop_eye(1, alive),
                getter=lambda: eye_procs_alive[1].value,
            ))

        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(g_pool.version)))
        general_settings.append(
            ui.Button("Restart with default settings", reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(True)

        # now that we have a proper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        if session_settings.get("eye1_process_alive", True):
            launch_eye_process(1, delay=0.6)
        if session_settings.get("eye0_process_alive", True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({"subject": "world_process.started"})
        logger.warning("Process started.")

        # Event loop
        while (not glfw.glfwWindowShouldClose(main_window)
               and not process_was_interrupted):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # "blacklisted" events that were already sent
            del events["pupil"]
            del events["gaze"]
            # delete if exists. More expensive than del, so only use it when key might not exist
            events.pop("annotation", None)

            # send new events to ipc:
            if "frame" in events:
                del events["frame"]  # send explicitly with frame publisher
            if "depth_frame" in events:
                del events["depth_frame"]
            if "audio_packets" in events:
                del events["audio_packets"]
            del events["dt"]  # no need to send this
            for data in events.values():
                assert isinstance(data, (list, tuple))
                for d in data:
                    ipc_pub.send(d)

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            glfw.glfwPollEvents()
            if window_should_update() and gl_utils.is_window_visible(
                    main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipboard is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for button, action, mods in user_input.buttons:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                glfw.glfwSwapBuffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)
        session_settings["eye0_process_alive"] = eye_procs_alive[0].value
        session_settings["eye1_process_alive"] = eye_procs_alive[1].value
        session_settings[
            "min_calibration_confidence"] = g_pool.min_calibration_confidence
        session_settings[
            "detection_mapping_mode"] = g_pool.detection_mapping_mode
        session_settings["audio_mode"] = audio.audio_mode

        if not hide_ui:
            glfw.glfwRestoreWindow(
                main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.glfwGetWindowPos(
                main_window)
            session_window_size = glfw.glfwGetWindowSize(main_window)
            if 0 not in session_window_size:
                session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Capture crashed with trace:\n{}".format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "world_process.stopped"})
        sleep(1.0)
Beispiel #58
0
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
          user_dir, version):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    #general imports
    from time import time, sleep
    import numpy as np
    import logging

    #networking
    import zmq
    import zmq_tools

    #zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                       ipc_sub_url,
                                       topics=('pupil', ))
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    #log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    #display
    import glfw
    from pyglui import ui, graph, cygl, __version__ as pyglui_version
    assert pyglui_version >= '1.0'
    from pyglui.cygl.utils import Named_Texture
    from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush, is_window_visible

    #monitoring
    import psutil

    # helpers/utils
    from version_utils import VersionFormat
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t, get_system_info
    from uvc import get_time_monotonic
    logger.info('Application Version: %s' % version)
    logger.info('System Info: %s' % get_system_info())

    # video sources
    from video_capture import InitialisationError, StreamError, Fake_Source, EndofVideoFileError, source_classes, manager_classes
    source_by_name = {src.class_name(): src for src in source_classes}
    import audio

    #trigger pupil detector cpp build:
    import pupil_detectors
    del pupil_detectors

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins
    from fixation_detector import Fixation_Detector_3D
    from recorder import Recorder
    from show_calibration import Show_Calibration
    from display_recent_gaze import Display_Recent_Gaze
    from time_sync import Time_Sync
    from pupil_remote import Pupil_Remote
    from pupil_groups import Pupil_Groups
    from surface_tracker import Surface_Tracker
    from log_display import Log_Display
    from annotations import Annotation_Capture
    from log_history import Log_History
    from frame_publisher import Frame_Publisher

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (0, 0)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (8, 31)
    else:
        scroll_factor = 1.0
        window_position_default = (0, 0)

    #g_pool holds variables for this process they are accesible to all plugins
    g_pool = Global_Container()
    g_pool.app = 'capture'
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.timebase = timebase
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url
    g_pool.eyes_are_alive = eyes_are_alive

    def get_timestamp():
        return get_time_monotonic() - g_pool.timebase.value

    g_pool.get_timestamp = get_timestamp
    g_pool.get_now = get_time_monotonic

    #manage plugins
    runtime_plugins = import_runtime_plugins(
        os.path.join(g_pool.user_dir, 'plugins'))
    user_launchable_plugins = [
        Pupil_Groups, Frame_Publisher, Show_Calibration, Pupil_Remote,
        Time_Sync, Surface_Tracker, Annotation_Capture, Log_History,
        Fixation_Detector_3D
    ] + runtime_plugins
    system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
    plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index, plugin_by_index))
    default_plugins = [('UVC_Manager', {}), ('Log_Display', {}),
                       ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}),
                       ('Screen_Marker_Calibration', {}), ('Recorder', {}),
                       ('Pupil_Remote', {}), ('Fixation_Detector_3D', {})]

    # Callback functions
    def on_resize(window, w, h):
        if is_window_visible(window):
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfw.glfwGetCursorPos(window)
        pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
        pos = denormalize(
            pos,
            (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfw.glfwGetFramebufferSize(window)[0] /
            glfw.glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    tick = delta_t()

    def get_dt():
        return next(tick)

    g_pool.on_frame_size_change = lambda new_size: None

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_world'))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()

    # Initialize capture
    default_settings = {
        'source_class_name':
        'UVC_Source',
        'preferred_names': [
            "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525",
            "C525", "C615", "C920", "C930e"
        ],
        'frame_size': (1280, 720),
        'frame_rate':
        30
    }
    settings = session_settings.get('capture_settings', default_settings)
    try:
        cap = source_by_name[settings['source_class_name']](g_pool, **settings)
    except (KeyError, InitialisationError) as e:
        if isinstance(e, KeyError):
            logger.warning(
                'Incompatible capture setting encountered. Falling back to fake source.'
            )
        cap = Fake_Source(g_pool, **settings)

    g_pool.iconified = False
    g_pool.detection_mapping_mode = session_settings.get(
        'detection_mapping_mode', '3d')
    g_pool.active_calibration_plugin = None
    g_pool.active_gaze_mapping_plugin = None
    g_pool.capture = cap
    g_pool.capture_manager = None

    audio.audio_mode = session_settings.get('audio_mode',
                                            audio.default_audio_mode)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def launch_eye_process(eye_id, delay=0):
        n = {
            'subject': 'eye_process.should_start.%s' % eye_id,
            'eye_id': eye_id,
            'delay': delay
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id}
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
        ipc_pub.notify(n)

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'set_detection_mapping_mode':
            if n['mode'] == '2d':
                if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name:
                    logger.warning(
                        "The gaze mapper is not supported in 2d mode. Please recalibrate."
                    )
                    g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
            g_pool.detection_mapping_mode = n['mode']
        elif subject == 'start_plugin':
            g_pool.plugins.add(plugin_by_name[n['name']],
                               args=n.get('args', {}))
        elif subject == 'eye_process.started':
            n = {
                'subject': 'set_detection_mapping_mode',
                'mode': g_pool.detection_mapping_mode
            }
            ipc_pub.notify(n)
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({
                'subject': 'meta.doc',
                'actor': g_pool.app,
                'doc': world.__doc__
            })
            for p in g_pool.plugins:
                if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify:
                    ipc_pub.notify({
                        'subject': 'meta.doc',
                        'actor': p.class_name,
                        'doc': p.on_notify.__doc__
                    })

    #window and gl setup
    glfw.glfwInit()
    width, height = session_settings.get('window_size', cap.frame_size)
    main_window = glfw.glfwCreateWindow(width, height, "World")
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-350, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface size'))
    general_settings.append(
        ui.Button(
            'Reset window size', lambda: glfw.glfwSetWindowSize(
                main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('audio_mode', audio, selection=audio.audio_modes))
    general_settings.append(
        ui.Selector('detection_mapping_mode',
                    g_pool,
                    label='detection & mapping mode',
                    setter=set_detection_mapping_mode,
                    selection=['2d', '3d']))
    general_settings.append(
        ui.Switch('eye0_process',
                  label='Detect eye 0',
                  setter=lambda alive: start_stop_eye(0, alive),
                  getter=lambda: eyes_are_alive[0].value))
    general_settings.append(
        ui.Switch('eye1_process',
                  label='Detect eye 1',
                  setter=lambda alive: start_stop_eye(1, alive),
                  getter=lambda: eyes_are_alive[1].value))

    selector_label = "Select to load"
    labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins]
    user_launchable_plugins.insert(0, selector_label)
    labels.insert(0, selector_label)
    general_settings.append(
        ui.Selector('Open plugin',
                    selection=user_launchable_plugins,
                    labels=labels,
                    setter=open_plugin,
                    getter=lambda: selector_label))

    general_settings.append(
        ui.Info_Text('Capture Version: %s' % g_pool.version))

    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))

    g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
    g_pool.capture.init_gui()

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')

    g_pool.sidebar.append(general_settings)
    g_pool.sidebar.append(g_pool.capture_selector_menu)
    g_pool.sidebar.append(g_pool.capture_source_menu)
    g_pool.sidebar.append(g_pool.calibration_menu)

    g_pool.gui.append(g_pool.sidebar)
    g_pool.gui.append(g_pool.quickbar)

    #plugins that are loaded based on user settings from previous session
    g_pool.plugins = Plugin_List(
        g_pool, plugin_by_name,
        session_settings.get('loaded_plugins', default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(
        0,
        ui.Selector(
            'active_calibration_plugin',
            getter=lambda: g_pool.active_calibration_plugin.__class__,
            selection=calibration_plugins,
            labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
            setter=open_plugin,
            label='Method'))

    #We add the capture selection menu, after a manager has been added:
    g_pool.capture_selector_menu.insert(
        0,
        ui.Selector('capture_manager',
                    setter=open_plugin,
                    getter=lambda: g_pool.capture_manager.__class__,
                    selection=manager_classes,
                    labels=[b.gui_name for b in manager_classes],
                    label='Manager'))

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfw.glfwSetKeyCallback(main_window, on_key)
    glfw.glfwSetCharCallback(main_window, on_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    # refresh speed settings
    glfw.glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = g_pool.get_timestamp()

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil0_graph = graph.Bar_Graph(max_val=1.0)
    pupil0_graph.pos = (260, 130)
    pupil0_graph.update_rate = 5
    pupil0_graph.label = "id0 conf: %0.2f"
    pupil1_graph = graph.Bar_Graph(max_val=1.0)
    pupil1_graph.pos = (380, 130)
    pupil1_graph.update_rate = 5
    pupil1_graph.label = "id1 conf: %0.2f"
    pupil_graphs = pupil0_graph, pupil1_graph

    if session_settings.get('eye1_process_alive', False):
        launch_eye_process(1, delay=0.6)
    if session_settings.get('eye0_process_alive', True):
        launch_eye_process(0, delay=0.3)

    ipc_pub.notify({'subject': 'world_process.started'})
    logger.warning('Process started.')

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            g_pool.capture.on_notify(n)
            for p in g_pool.plugins:
                p.on_notify(n)

        # Get an image from the grabber
        try:
            frame = g_pool.capture.get_frame()
        except StreamError as e:
            prev_settings = g_pool.capture.settings
            g_pool.capture.deinit_gui()
            g_pool.capture.cleanup()
            g_pool.capture = None
            prev_settings[
                'info_text'] = "'%s' disconnected." % prev_settings['name']
            g_pool.capture = Fake_Source(g_pool, **prev_settings)
            g_pool.capture.init_gui()
            ipc_pub.notify({'subject': 'recording.should_stop'})
            logger.error("Error getting frame. Falling back to Fake source.")
            logger.debug("Caught error: %s" % e)
            sleep(.2)
            continue
        except EndofVideoFileError:
            logger.warning("Video file is done. Rewinding")
            g_pool.capture.seek_to_frame(0)
            continue

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        #a dictionary that allows plugins to post and read events
        events = {}
        #report time between now and the last loop interation
        events['dt'] = get_dt()

        recent_pupil_data = []
        recent_gaze_data = []

        while pupil_sub.new_data:
            t, p = pupil_sub.recv()
            pupil_graphs[p['id']].add(p['confidence'])
            recent_pupil_data.append(p)
            new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p)
            for g in new_gaze_data:
                gaze_pub.send('gaze', g)
            recent_gaze_data += new_gaze_data

        events['pupil_positions'] = recent_pupil_data
        events['gaze_positions'] = recent_gaze_data

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        #send new events to ipc:
        del events['pupil_positions']  #already on the wire
        del events['gaze_positions']  #send earlier in this loop
        del events['dt']  #no need to send this
        for topic, data in events.iteritems():
            assert (isinstance(data, (list, tuple)))
            for d in data:
                ipc_pub.send(topic, d)

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        if is_window_visible(main_window):
            g_pool.image_tex.update_from_frame(frame)
            glFlush()
        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height, frame.width, 3))
        # render visual feedback from loaded plugins

        if is_window_visible(main_window):
            g_pool.capture.gl_display()
            for p in g_pool.plugins:
                p.gl_display()

            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil0_graph.draw()
            pupil1_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings['eye0_process_alive'] = eyes_are_alive[0].value
    session_settings['eye1_process_alive'] = eyes_are_alive[1].value
    session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
    session_settings['audio_mode'] = audio.audio_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()

    g_pool.capture.deinit_gui()
    g_pool.capture.cleanup()

    #shut down eye processes:
    stop_eye_process(0)
    stop_eye_process(1)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'world_process.stopped'})

    #shut down launcher
    n = {'subject': 'launcher_process.should_stop'}
    ipc_pub.notify(n)
    zmq_ctx.destroy()
Beispiel #59
0
def eye(g_pool,cap_src,cap_size,rx_from_world,eye_id=0):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye%s.log'%eye_id),mode='w')
    # fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level+10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)


    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (600,300*eye_id)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (600,31+300*eye_id)
    else:
        scroll_factor = 1.0
        window_position_default = (600,300*eye_id)


    # Callback functions
    def on_resize(window,w, h):
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
        w,h = w*hdpi_factor, h*hdpi_factor
        g_pool.gui.update_window(w,h)
        graph.adjust_size(w,h)
        adjust_gl_view(w,h)
        # for p in g_pool.plugins:
            # p.on_window_resize(window,w,h)
        glfwMakeContextCurrent(active_window)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)


    def on_char(window,char):
        g_pool.gui.update_char(char)


    def on_button(window,button, action, mods):
        if g_pool.display_mode == 'roi':
            if action == GLFW_RELEASE and u_r.active_edit_pt:
                u_r.active_edit_pt = False
                return # if the roi interacts we dont what the gui to interact as well
            elif action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos,glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
                if u_r.mouse_over_edit_pt(pos,u_r.handle_size+40,u_r.handle_size+40):
                    return # if the roi interacts we dont what the gui to interact as well

        g_pool.gui.update_button(button,action,mods)



    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

        if u_r.active_edit_pt:
            pos = normalize((x,y),glfwGetWindowSize(main_window))
            if g_pool.flip:
                pos = 1-pos[0],1-pos[1]
            pos = denormalize(pos,(frame.width,frame.height) )
            u_r.move_vertex(u_r.active_pt_idx,pos)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*scroll_factor)

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')


    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
    if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()
    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    cap.frame_size = cap_size
    cap.frame_rate = 90 #default
    cap.settings = session_settings.get('capture_settings',{})


    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    g_pool.capture = cap
    g_pool.flip = session_settings.get('flip',False)
    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.window_size = session_settings.get('window_size',1.)
    g_pool.display_mode = session_settings.get('display_mode','camera_image')
    g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
                                'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."}
    # g_pool.draw_pupil = session_settings.get('draw_pupil',True)

    u_r = UIRoi(frame.img.shape)
    u_r.set(session_settings.get('roi',u_r.get()))

    writer = None

    pupil_detector = Canny_Detector(g_pool)


    # UI callback functions
    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()


    def set_display_mode_info(val):
        g_pool.display_mode = val
        g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]


    window_pos = session_settings.get('window_position',window_position_default)
    width,height = session_settings.get('window_size',(frame.width, frame.height))

    # Initialize glfw
    glfwInit()
    if g_pool.binocular:
        title = "Binocular eye %s"%eye_id
    else:
        title = 'Eye'
    main_window = glfwCreateWindow(width,height, title, None, None)

    glfwMakeContextCurrent(main_window)
    cygl_init()

    # Register callbacks main_window
    glfwSetWindowSizeCallback(main_window,on_resize)
    glfwSetWindowCloseCallback(main_window,on_close)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = create_named_texture(frame.img.shape)
    update_named_texture(g_pool.image_tex,frame.img)

    # refresh speed settings
    glfwSwapInterval(0)
    glfwSetWindowPos(main_window,window_pos[0],window_pos[1])


    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
    general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
    general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
    general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
    g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
    general_settings.append(g_pool.display_mode_info)
    g_pool.sidebar.append(general_settings)

    g_pool.gui.append(g_pool.sidebar)
    g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))


    # let the camera add its GUI
    g_pool.capture.init_gui(g_pool.sidebar)

    # let detector add its GUI
    pupil_detector.init_gui(g_pool.sidebar)

    # load last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config',{})


    #set the last saved window size
    on_resize(main_window, *glfwGetWindowSize(main_window))

    #set up performance graphs
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    # Event loop
    while not g_pool.quit.value:
        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break


        #update performace graphs
        t = frame.timestamp
        dt,ts = t-ts,t
        try:
            fps_graph.add(1./dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()


        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if rx_from_world.poll():
            command,raw_mode = rx_from_world.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %s"%record_path)
                video_path = os.path.join(record_path, "eye%s.mkv"%eye_id)
                timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
                if raw_mode:
                    writer = JPEG_Dumper(video_path)
                else:
                    writer = CV_Writer(video_path,float(cap.frame_rate), cap.frame_size)
                timestamps = []
            else:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path,np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write_video_frame(frame)
            timestamps.append(frame.timestamp)


        # pupil ellipse detection
        result = pupil_detector.detect(frame,user_roi=u_r,visualize=g_pool.display_mode == 'algorithm')
        result['id'] = eye_id
        # stream the result
        g_pool.pupil_queue.put(result)


        # GL drawing
        glfwMakeContextCurrent(main_window)
        clear_gl_screen()

        # switch to work in normalized coordinate space
        if g_pool.display_mode == 'algorithm':
            update_named_texture(g_pool.image_tex,frame.img)
        elif g_pool.display_mode in ('camera_image','roi'):
            update_named_texture(g_pool.image_tex,frame.gray)
        else:
            pass

        make_coord_system_norm_based(g_pool.flip)
        draw_named_texture(g_pool.image_tex)
        # switch to work in pixel space
        make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)

        if result['confidence'] >0:
            if result.has_key('axes'):
                pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
                                        (int(result['axes'][0]/2),int(result['axes'][1]/2)),
                                        int(result['angle']),0,360,15)
                cygl_draw_polyline(pts,1,cygl_rgba(1.,0,0,.5))
            cygl_draw_points([result['center']],size=20,color=cygl_rgba(1.,0.,0.,.5),sharpness=1.)

        # render graphs
        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        graph.pop_view()

        # render GUI
        g_pool.gui.update()

        #render the ROI
        if g_pool.display_mode == 'roi':
            u_r.draw(g_pool.gui.scale)

        #update screen
        glfwSwapBuffers(main_window)
        glfwPollEvents()


    # END while running

    # in case eye recording was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path,np.asarray(timestamps))


    # save session persistent settings
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['roi'] = u_r.get()
    session_settings['flip'] = g_pool.flip
    session_settings['display_mode'] = g_pool.display_mode
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    pupil_detector.cleanup()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    #flushing queue in case world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")