Ejemplo n.º 1
0
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """
    is_alive = Is_Alive_Manager(is_alive_flag)
    with is_alive:
        import logging
        # Set up root logger for this process before doing imports of logged modules.
        logger = logging.getLogger()
        logger.setLevel(logging.INFO)
        # remove inherited handlers
        logger.handlers = []
        # create file handler which logs even debug messages
        fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w')
        # fh.setLevel(logging.DEBUG)
        # create console handler with a higher log level
        ch = logging.StreamHandler()
        ch.setLevel(logger.level+10)
        # create formatter and add it to the handlers
        formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
        ch.setFormatter(formatter)
        # add the handlers to the logger
        logger.addHandler(fh)
        logger.addHandler(ch)
        #silence noisy modules
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        # create logger for the context of this function
        logger = logging.getLogger(__name__)


        # We deferr the imports becasue of multiprocessing.
        # Otherwise the world process each process also loads the other imports.

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui,graph,cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
        import OpenGL.GL as gl
        from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based
        from ui_roi import UIRoi
        #monitoring
        import psutil
        import math


        # helpers/utils
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
        from av_writer import JPEG_Writer,AV_Writer

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}



        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600,300*eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600,31+300*eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600,300*eye_id)


        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.pupil_queue = pupil_queue
        g_pool.timebase = timebase


        # Callback functions
        def on_resize(window,w, h):
            if not g_pool.iconified:
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w,h)
                graph.adjust_size(w,h)
                adjust_gl_view(w,h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key,scancode,action,mods)

        def on_char(window,char):
            g_pool.gui.update_char(char)

        def on_iconify(window,iconified):
            g_pool.iconified = iconified

        def on_button(window,button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1-pos[0],1-pos[1]
                    pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
                        return # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button,action,mods)



        def on_pos(window,x, y):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height) )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)

        def on_scroll(window,x,y):
            g_pool.gui.update_scroll(x,y*scroll_factor)


        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
        if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
            logger.info("Session setting are from older version of this app. I will not use those.")
            session_settings.clear()
        # Initialize capture
        cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
        default_settings = {'frame_size':(640,480),'frame_rate':60}
        previous_settings = session_settings.get('capture_settings',None)
        if previous_settings and previous_settings['name'] == cap.name:
            cap.settings = previous_settings
        else:
            cap.settings = default_settings


        # Test capture
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Could not retrieve image from capture")
            cap.close()
            return

        #signal world that we are ready to go
        # pipe_to_world.send('eye%s process ready'%eye_id)

        # any object we attach to the g_pool object *from now on* will only be visible to this process!
        # vars should be declared here to make them visible to the code reader.
        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.flip = session_settings.get('flip',False)
        g_pool.display_mode = session_settings.get('display_mode','camera_image')
        g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                    'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
                                    'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}


        g_pool.u_r = UIRoi(frame.img.shape)
        g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get()))


        def on_frame_size_change(new_size):
            g_pool.u_r = UIRoi((new_size[1],new_size[0]))

        cap.on_frame_size_change = on_frame_size_change

        writer = None

        pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
        last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()


        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]


        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)


        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s"%eye_id
        width,height = session_settings.get('window_size',(frame.width, frame.height))
        main_window = glfw.glfwCreateWindow(width,height, title, None, None)
        window_pos = session_settings.get('window_position',window_position_default)
        glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_frame(frame)
        glfw.glfwSwapInterval(0)

        sphere  = Sphere(20)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale',1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
        general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
        general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
        general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
        g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.sidebar.append(general_settings)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
        general_settings.append(detector_selector)

        # let detector add its GUI
        g_pool.pupil_detector.init_gui(g_pool.sidebar)
        # let the camera add its GUI
        g_pool.capture.init_gui(g_pool.sidebar)


        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
        glfw.glfwSetKeyCallback(main_window,on_key)
        glfw.glfwSetCharCallback(main_window,on_char)
        glfw.glfwSetMouseButtonCallback(main_window,on_button)
        glfw.glfwSetCursorPosCallback(main_window,on_pos)
        glfw.glfwSetScrollCallback(main_window,on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))


        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config',{})


        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = frame.timestamp

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20,130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140,130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"


        #create a timer to control window update frequency
        window_update_timer = timer(1/60.)
        def window_should_update():
            return next(window_update_timer)


        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if pipe_to_world.poll():
                cmd = pipe_to_world.recv()
                if cmd == 'Exit':
                    break
                elif cmd == "Ping":
                    pipe_to_world.send("Pong")
                    command = None
                else:
                    command,payload = cmd
                if command == 'Set_Detection_Mapping_Mode':
                    if payload == '3d':
                        if not isinstance(g_pool.pupil_detector,Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only  = True
                    else:
                        set_detector(Detector_2D)
                        detector_selector.read_only = False

            else:
                command = None



            # Get an image from the grabber
            try:
                frame = cap.get_frame()
            except CameraCaptureError:
                logger.error("Capture from Camera Failed. Stopping.")
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                cap.seek_to_frame(0)
                frame = cap.get_frame()


            #update performace graphs
            t = frame.timestamp
            dt,ts = t-ts,t
            try:
                fps_graph.add(1./dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()


            ###  RECORDING of Eye Video (on demand) ###
            # Setup variables and lists for recording
            if 'Rec_Start' == command:
                record_path,raw_mode = payload
                logger.info("Will save eye video to: %s"%record_path)
                timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                    writer = JPEG_Writer(video_path,cap.frame_rate)
                else:
                    video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                    writer = AV_Writer(video_path,cap.frame_rate)
                timestamps = []
            elif 'Rec_Stop' == command:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path,np.asarray(timestamps))
                del timestamps

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)


            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            g_pool.pupil_queue.put(result)

            # GL drawing
            if window_should_update():
                if not g_pool.iconified:
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image','roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    window_size =  glfw.glfwGetWindowSize(main_window)
                    make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)

                    if result['method'] == '3d c++':

                        eye_ball = result['projected_sphere']
                        try:
                            pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
                                                (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
                                                int(eye_ball['angle']),0,360,8)
                        except ValueError as e:
                            pass
                        else:
                            draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )

                    if result['confidence'] >0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
                                            (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
                                            int(result['ellipse']['angle']),0,360,15)
                            confidence = result['confidence'] * 0.7 #scale it a little
                            draw_polyline(pts,1,RGBA(1.,0,0,confidence))
                            draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize() #detector decides if we visualize or not


        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path,np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window) #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = g_pool.version
        session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        cap.close()


        logger.debug("Process done")
Ejemplo n.º 2
0
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version,
        eye_id, cap_src):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """
    is_alive = Is_Alive_Manager(is_alive_flag)
    with is_alive:
        import logging
        # Set up root logger for this process before doing imports of logged modules.
        logger = logging.getLogger()
        logger.setLevel(logging.INFO)
        # remove inherited handlers
        logger.handlers = []
        # create file handler which logs even debug messages
        fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id),
                                 mode='w')
        # fh.setLevel(logging.DEBUG)
        # create console handler with a higher log level
        ch = logging.StreamHandler()
        ch.setLevel(logger.level + 10)
        # create formatter and add it to the handlers
        formatter = logging.Formatter(
            'Eye' + str(eye_id) +
            ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        formatter = logging.Formatter(
            'EYE' + str(eye_id) +
            ' Process [%(levelname)s] %(name)s : %(message)s')
        ch.setFormatter(formatter)
        # add the handlers to the logger
        logger.addHandler(fh)
        logger.addHandler(ch)
        #silence noisy modules
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logging.getLogger("libav").setLevel(logging.ERROR)
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        # We deferr the imports becasue of multiprocessing.
        # Otherwise the world process each process also loads the other imports.

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture
        from OpenGL.GL import GL_LINE_LOOP
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based
        from ui_roi import UIRoi
        #monitoring
        import psutil

        # helpers/utils
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
        from av_writer import JPEG_Writer, AV_Writer

        # Pupil detectors
        from pupil_detectors import Canny_Detector, Detector_2D, Detector_3D
        pupil_detectors = {
            Canny_Detector.__name__: Canny_Detector,
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D
        }

        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600, 31 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.pupil_queue = pupil_queue
        g_pool.timebase = timebase

        # Callback functions
        def on_resize(window, w, h):
            if not g_pool.iconified:
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w, h)
                graph.adjust_size(w, h)
                adjust_gl_view(w, h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_button(window, button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return  # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    pos = denormalize(
                        pos,
                        (frame.width, frame.height))  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                            pos, g_pool.u_r.handle_size + 40,
                            g_pool.u_r.handle_size + 40):
                        return  # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, (frame.width, frame.height))
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
        if session_settings.get("version",
                                VersionFormat('0.0')) < g_pool.version:
            logger.info(
                "Session setting are from older version of this app. I will not use those."
            )
            session_settings.clear()
        # Initialize capture
        cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
        default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
        previous_settings = session_settings.get('capture_settings', None)
        if previous_settings and previous_settings['name'] == cap.name:
            cap.settings = previous_settings
        else:
            cap.settings = default_settings

        # Test capture
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Could not retrieve image from capture")
            cap.close()
            return

        #signal world that we are ready to go
        # pipe_to_world.send('eye%s process ready'%eye_id)

        # any object we attach to the g_pool object *from now on* will only be visible to this process!
        # vars should be declared here to make them visible to the code reader.
        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.flip = session_settings.get('flip', False)
        g_pool.display_mode = session_settings.get('display_mode',
                                                   'camera_image')
        g_pool.display_mode_info_text = {
            'camera_image':
            "Raw eye camera image. This uses the least amount of CPU power",
            'roi':
            "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
            'algorithm':
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
        }

        g_pool.u_r = UIRoi(frame.img.shape)
        g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))

        def on_frame_size_change(new_size):
            g_pool.u_r = UIRoi((new_size[1], new_size[0]))

        cap.on_frame_size_change = on_frame_size_change

        writer = None

        pupil_detector_settings = session_settings.get(
            'pupil_detector_settings', None)
        last_pupil_detector = pupil_detectors[session_settings.get(
            'last_pupil_detector', Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,
                                                    pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)

        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s" % eye_id
        width, height = session_settings.get('window_size',
                                             (frame.width, frame.height))
        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_frame(frame)
        glfw.glfwSwapInterval(0)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale', 1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                           pos=(-300, 0),
                                           size=(0, 0),
                                           header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(
            ui.Slider('scale',
                      g_pool.gui,
                      setter=set_scale,
                      step=.05,
                      min=1.,
                      max=2.5,
                      label='Interface Size'))
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, frame.width, frame.height)))
        general_settings.append(
            ui.Switch('flip', g_pool, label='Flip image display'))
        general_settings.append(
            ui.Selector('display_mode',
                        g_pool,
                        setter=set_display_mode_info,
                        selection=['camera_image', 'roi', 'algorithm'],
                        labels=['Camera Image', 'ROI', 'Algorithm'],
                        label="Mode"))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.sidebar.append(general_settings)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector(
            'pupil_detector',
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Canny_Detector, Detector_2D, Detector_3D],
            labels=[
                'Python 2D detector', 'C++ 2d detector', 'C++ 3d detector'
            ],
            label="Detection method")
        general_settings.append(detector_selector)

        # let detector add its GUI
        g_pool.pupil_detector.init_gui(g_pool.sidebar)
        # let the camera add its GUI
        g_pool.capture.init_gui(g_pool.sidebar)

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_key)
        glfw.glfwSetCharCallback(main_window, on_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = frame.timestamp

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"

        #create a timer to control window update frequency
        window_update_timer = timer(1 / 60.)

        def window_should_update():
            return next(window_update_timer)

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if pipe_to_world.poll():
                cmd = pipe_to_world.recv()
                if cmd == 'Exit':
                    break
                elif cmd == "Ping":
                    pipe_to_world.send("Pong")
                    command = None
                else:
                    command, payload = cmd
                if command == 'Set_Detection_Mapping_Mode':
                    if payload == '3d':
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    else:
                        set_detector(Detector_2D)
                        detector_selector.read_only = False

            else:
                command = None

            # Get an image from the grabber
            try:
                frame = cap.get_frame()
            except CameraCaptureError:
                logger.error("Capture from Camera Failed. Stopping.")
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                cap.seek_to_frame(0)
                frame = cap.get_frame()

            #update performace graphs
            t = frame.timestamp
            dt, ts = t - ts, t
            try:
                fps_graph.add(1. / dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()

            ###  RECORDING of Eye Video (on demand) ###
            # Setup variables and lists for recording
            if 'Rec_Start' == command:
                record_path, raw_mode = payload
                logger.info("Will save eye video to: %s" % record_path)
                timestamps_path = os.path.join(record_path,
                                               "eye%s_timestamps.npy" % eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = JPEG_Writer(video_path, cap.frame_rate)
                else:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = AV_Writer(video_path, cap.frame_rate)
                timestamps = []
            elif 'Rec_Stop' == command:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)

            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(
                frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            g_pool.pupil_queue.put(result)

            # GL drawing
            if window_should_update():
                if not g_pool.iconified:
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image', 'roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()
                    # switch to work in pixel space
                    make_coord_system_pixel_based(
                        (frame.height, frame.width, 3), g_pool.flip)

                    if result['confidence'] > 0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly(
                                (int(result['ellipse']['center'][0]),
                                 int(result['ellipse']['center'][1])),
                                (int(result['ellipse']['axes'][0] / 2),
                                 int(result['ellipse']['axes'][1] / 2)),
                                int(result['ellipse']['angle']), 0, 360, 15)
                            draw_polyline(pts, 1, RGBA(1., 0, 0, .5))
                        draw_points([result['ellipse']['center']],
                                    size=20,
                                    color=RGBA(1., 0., 0., .5),
                                    sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize(
                )  #detector decides if we visualize or not

        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path, np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = g_pool.version
        session_settings[
            'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        cap.close()

        logger.debug("Process done")
Ejemplo n.º 3
0
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir,
                                          'eye%s.log' % eye_id),
                             mode='w')
    # fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level + 10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        'Eye' + str(eye_id) +
        ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter(
        'EYE' + str(eye_id) +
        ' Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (600, 300 * eye_id)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (600, 31 + 300 * eye_id)
    else:
        scroll_factor = 1.0
        window_position_default = (600, 300 * eye_id)

    # Callback functions
    def on_resize(window, w, h):
        if not g_pool.iconified:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(window)
            g_pool.gui.update_window(w, h)
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            glfwMakeContextCurrent(active_window)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_button(window, button, action, mods):
        if g_pool.display_mode == 'roi':
            if action == GLFW_RELEASE and u_r.active_edit_pt:
                u_r.active_edit_pt = False
                return  # if the roi interacts we dont what the gui to interact as well
            elif action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos, glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(
                    pos, (frame.width, frame.height))  # Position in img pixels
                if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
                                          u_r.handle_size + 40):
                    return  # if the roi interacts we dont what the gui to interact as well

        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

        if u_r.active_edit_pt:
            pos = normalize((x, y), glfwGetWindowSize(main_window))
            if g_pool.flip:
                pos = 1 - pos[0], 1 - pos[1]
            pos = denormalize(pos, (frame.width, frame.height))
            u_r.move_vertex(u_r.active_pt_idx, pos)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()
    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size': cap_size, 'frame_rate': 30}
    previous_settings = session_settings.get('capture_settings', None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    #signal world that we are ready to go
    pipe_to_world.send('eye%s process ready' % eye_id)

    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.flip = session_settings.get('flip', False)
    g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
    g_pool.display_mode_info_text = {
        'camera_image':
        "Raw eye camera image. This uses the least amount of CPU power",
        'roi':
        "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
        'algorithm':
        "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
    }
    # g_pool.draw_pupil = session_settings.get('draw_pupil',True)

    u_r = UIRoi(frame.img.shape)
    u_r.set(session_settings.get('roi', u_r.get()))

    writer = None

    pupil_detector = Canny_Detector(g_pool)

    # UI callback functions
    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def set_display_mode_info(val):
        g_pool.display_mode = val
        g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

    # Initialize glfw
    glfwInit()
    if g_pool.binocular:
        title = "Binocular eye %s" % eye_id
    else:
        title = 'Eye'
    width, height = session_settings.get('window_size',
                                         (frame.width, frame.height))
    main_window = glfwCreateWindow(width, height, title, None, None)
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl_init()

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    glfwSwapInterval(0)

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-300, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface Size'))
    general_settings.append(
        ui.Button(
            'Reset window size',
            lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('display_mode',
                    g_pool,
                    setter=set_display_mode_info,
                    selection=['camera_image', 'roi', 'algorithm'],
                    labels=['Camera Image', 'ROI', 'Algorithm'],
                    label="Mode"))
    general_settings.append(
        ui.Switch('flip', g_pool, label='Flip image display'))
    g_pool.display_mode_info = ui.Info_Text(
        g_pool.display_mode_info_text[g_pool.display_mode])
    general_settings.append(g_pool.display_mode_info)
    g_pool.sidebar.append(general_settings)
    g_pool.gui.append(g_pool.sidebar)
    # let the camera add its GUI
    g_pool.capture.init_gui(g_pool.sidebar)
    # let detector add its GUI
    pupil_detector.init_gui(g_pool.sidebar)

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetWindowCloseCallback(main_window, on_close)
    glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)

    #set the last saved window size
    on_resize(main_window, *glfwGetWindowSize(main_window))

    # load last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performance graphs
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    #create a timer to control window update frequency
    window_update_timer = timer(1 / 60.)

    def window_should_update():
        return next(window_update_timer)

    # Event loop
    while not g_pool.quit.value:
        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if pipe_to_world.poll():
            command, raw_mode = pipe_to_world.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %s" % record_path)
                timestamps_path = os.path.join(record_path,
                                               "eye%s_timestamps.npy" % eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = JPEG_Writer(video_path, cap.frame_rate)
                else:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = AV_Writer(video_path, cap.frame_rate)
                timestamps = []
            else:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write_video_frame(frame)
            timestamps.append(frame.timestamp)

        # pupil ellipse detection
        result = pupil_detector.detect(
            frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
        result['id'] = eye_id
        # stream the result
        g_pool.pupil_queue.put(result)

        # GL drawing
        if window_should_update():
            if not g_pool.iconified:
                glfwMakeContextCurrent(main_window)
                clear_gl_screen()

                # switch to work in normalized coordinate space
                if g_pool.display_mode == 'algorithm':
                    g_pool.image_tex.update_from_ndarray(frame.img)
                elif g_pool.display_mode in ('camera_image', 'roi'):
                    g_pool.image_tex.update_from_ndarray(frame.gray)
                else:
                    pass

                make_coord_system_norm_based(g_pool.flip)
                g_pool.image_tex.draw()
                # switch to work in pixel space
                make_coord_system_pixel_based((frame.height, frame.width, 3),
                                              g_pool.flip)

                if result['confidence'] > 0:
                    if result.has_key('axes'):
                        pts = cv2.ellipse2Poly((int(
                            result['center'][0]), int(result['center'][1])),
                                               (int(result['axes'][0] / 2),
                                                int(result['axes'][1] / 2)),
                                               int(result['angle']), 0, 360,
                                               15)
                        cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
                    cygl_draw_points([result['center']],
                                     size=20,
                                     color=cygl_rgba(1., 0., 0., .5),
                                     sharpness=1.)

                # render graphs
                graph.push_view()
                fps_graph.draw()
                cpu_graph.draw()
                graph.pop_view()

                # render GUI
                g_pool.gui.update()

                #render the ROI
                if g_pool.display_mode == 'roi':
                    u_r.draw(g_pool.gui.scale)

                #update screen
                glfwSwapBuffers(main_window)
            glfwPollEvents()

    # END while running

    # in case eye recording was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path, np.asarray(timestamps))

    glfwRestoreWindow(main_window)  #need to do this for windows os
    # save session persistent settings
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['roi'] = u_r.get()
    session_settings['flip'] = g_pool.flip
    session_settings['display_mode'] = g_pool.display_mode
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    pupil_detector.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    #flushing queue in case world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")
Ejemplo n.º 4
0
def eye(g_pool,cap_src,cap_size,pipe_to_world,eye_id=0):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye%s.log'%eye_id),mode='w')
    # fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level+10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)


    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (600,300*eye_id)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (600,31+300*eye_id)
    else:
        scroll_factor = 1.0
        window_position_default = (600,300*eye_id)


    # Callback functions
    def on_resize(window,w, h):
        if not g_pool.iconified:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(window)
            g_pool.gui.update_window(w,h)
            graph.adjust_size(w,h)
            adjust_gl_view(w,h)
            glfwMakeContextCurrent(active_window)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)

    def on_iconify(window,iconified):
        g_pool.iconified = iconified

    def on_button(window,button, action, mods):
        if g_pool.display_mode == 'roi':
            if action == GLFW_RELEASE and u_r.active_edit_pt:
                u_r.active_edit_pt = False
                return # if the roi interacts we dont what the gui to interact as well
            elif action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos,glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
                if u_r.mouse_over_edit_pt(pos,u_r.handle_size+40,u_r.handle_size+40):
                    return # if the roi interacts we dont what the gui to interact as well

        g_pool.gui.update_button(button,action,mods)



    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

        if u_r.active_edit_pt:
            pos = normalize((x,y),glfwGetWindowSize(main_window))
            if g_pool.flip:
                pos = 1-pos[0],1-pos[1]
            pos = denormalize(pos,(frame.width,frame.height) )
            u_r.move_vertex(u_r.active_pt_idx,pos)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*scroll_factor)

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')


    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
    if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()
    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size':cap_size,'frame_rate':30}
    previous_settings = session_settings.get('capture_settings',None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    #signal world that we are ready to go
    pipe_to_world.send('eye%s process ready'%eye_id)

    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.flip = session_settings.get('flip',False)
    g_pool.display_mode = session_settings.get('display_mode','camera_image')
    g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
                                'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."}
    # g_pool.draw_pupil = session_settings.get('draw_pupil',True)

    u_r = UIRoi(frame.img.shape)
    u_r.set(session_settings.get('roi',u_r.get()))

    writer = None

    pupil_detector = Canny_Detector(g_pool)


    # UI callback functions
    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()


    def set_display_mode_info(val):
        g_pool.display_mode = val
        g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]


    # Initialize glfw
    glfwInit()
    if g_pool.binocular:
        title = "Binocular eye %s"%eye_id
    else:
        title = 'Eye'
    width,height = session_settings.get('window_size',(frame.width, frame.height))
    main_window = glfwCreateWindow(width,height, title, None, None)
    window_pos = session_settings.get('window_position',window_position_default)
    glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl_init()

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = create_named_texture(frame.img.shape)
    update_named_texture(g_pool.image_tex,frame.img)
    glfwSwapInterval(0)


    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
    general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
    general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
    general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
    g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
    general_settings.append(g_pool.display_mode_info)
    g_pool.sidebar.append(general_settings)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
    # let the camera add its GUI
    g_pool.capture.init_gui(g_pool.sidebar)
    # let detector add its GUI
    pupil_detector.init_gui(g_pool.sidebar)

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window,on_resize)
    glfwSetWindowCloseCallback(main_window,on_close)
    glfwSetWindowIconifyCallback(main_window,on_iconify)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)

    #set the last saved window size
    on_resize(main_window, *glfwGetWindowSize(main_window))


    # load last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config',{})


    #set up performance graphs
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"


    #create a timer to control window update frequency
    window_update_timer = timer(1/60.)
    def window_should_update():
        return next(window_update_timer)


    # Event loop
    while not g_pool.quit.value:
        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break


        #update performace graphs
        t = frame.timestamp
        dt,ts = t-ts,t
        try:
            fps_graph.add(1./dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()


        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if pipe_to_world.poll():
            command,raw_mode = pipe_to_world.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %s"%record_path)
                timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
                if raw_mode and hasattr(frame,'jpeg_buffer') :
                    video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                    writer = JPEG_Writer(video_path,cap.frame_rate)
                else:
                    video_path = os.path.join(record_path, "eye%s.mkv"%eye_id)
                    writer = CV_Writer(video_path,float(cap.frame_rate), cap.frame_size)
                timestamps = []
            else:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path,np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write_video_frame(frame)
            timestamps.append(frame.timestamp)


        # pupil ellipse detection
        result = pupil_detector.detect(frame,user_roi=u_r,visualize=g_pool.display_mode == 'algorithm')
        result['id'] = eye_id
        # stream the result
        g_pool.pupil_queue.put(result)

        # GL drawing
        if window_should_update():
            if not g_pool.iconified:
                glfwMakeContextCurrent(main_window)
                clear_gl_screen()

                # switch to work in normalized coordinate space
                if g_pool.display_mode == 'algorithm':
                    update_named_texture(g_pool.image_tex,frame.img)
                elif g_pool.display_mode in ('camera_image','roi'):
                    update_named_texture(g_pool.image_tex,frame.gray)
                else:
                    pass

                make_coord_system_norm_based(g_pool.flip)
                draw_named_texture(g_pool.image_tex)
                # switch to work in pixel space
                make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)

                if result['confidence'] >0:
                    if result.has_key('axes'):
                        pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
                                                (int(result['axes'][0]/2),int(result['axes'][1]/2)),
                                                int(result['angle']),0,360,15)
                        cygl_draw_polyline(pts,1,cygl_rgba(1.,0,0,.5))
                    cygl_draw_points([result['center']],size=20,color=cygl_rgba(1.,0.,0.,.5),sharpness=1.)

                # render graphs
                graph.push_view()
                fps_graph.draw()
                cpu_graph.draw()
                graph.pop_view()

                # render GUI
                g_pool.gui.update()

                #render the ROI
                if g_pool.display_mode == 'roi':
                    u_r.draw(g_pool.gui.scale)

                #update screen
                glfwSwapBuffers(main_window)
            glfwPollEvents()

    # END while running

    # in case eye recording was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path,np.asarray(timestamps))

    glfwRestoreWindow(main_window) #need to do this for windows os
    # save session persistent settings
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['roi'] = u_r.get()
    session_settings['flip'] = g_pool.flip
    session_settings['display_mode'] = g_pool.display_mode
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    pupil_detector.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    #flushing queue in case world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")
Ejemplo n.º 5
0
def eye(timebase,
        is_alive_flag,
        ipc_pub_url,
        ipc_sub_url,
        ipc_push_url,
        user_dir,
        version,
        eye_id,
        overwrite_cap_settings=None):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools
    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):

        #logging setup
        import logging
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logger = logging.getLogger()
        logger.handlers = []
        logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
        import OpenGL.GL as gl
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, make_coord_system_eye_camera_based, is_window_visible
        from ui_roi import UIRoi
        #monitoring
        import psutil
        import math

        # helpers/utils
        from uvc import get_time_monotonic, StreamError
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from av_writer import JPEG_Writer, AV_Writer

        from video_capture import InitialisationError, StreamError, Fake_Source, EndofVideoFileError, source_classes, manager_classes
        source_by_name = {src.class_name(): src for src in source_classes}

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D
        }

        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600, 31 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window, w, h):
            if is_window_visible(window):
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w, h)
                graph.adjust_size(w, h)
                adjust_gl_view(w, h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_button(window, button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return  # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    pos = denormalize(
                        pos,
                        (frame.width, frame.height))  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                            pos, g_pool.u_r.handle_size + 40,
                            g_pool.u_r.handle_size + 40):
                        return  # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, (frame.width, frame.height))
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
        if session_settings.get("version",
                                VersionFormat('0.0')) < g_pool.version:
            logger.info(
                "Session setting are from older version of this app. I will not use those."
            )
            session_settings.clear()

        capture_manager_settings = session_settings.get(
            'capture_manager_settings', ('UVC_Manager', {}))

        if eye_id == 0:
            cap_src = [
                "Pupil Cam1 ID0", "HD-6000", "Integrated Camera",
                "HD USB Camera", "USB 2.0 Camera"
            ]
        else:
            cap_src = ["Pupil Cam1 ID1", "HD-6000", "Integrated Camera"]

        # Initialize capture
        default_settings = {
            'source_class_name': 'UVC_Source',
            'preferred_names': cap_src,
            'frame_size': (640, 480),
            'frame_rate': 90
        }
        settings = overwrite_cap_settings or session_settings.get(
            'capture_settings', default_settings)
        try:
            cap = source_by_name[settings['source_class_name']](g_pool,
                                                                **settings)
        except (KeyError, InitialisationError) as e:
            if isinstance(e, KeyError):
                logger.warning(
                    'Incompatible capture setting encountered. Falling back to fake source.'
                )
            cap = Fake_Source(g_pool, **settings)

        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get('flip', False)
        g_pool.display_mode = session_settings.get('display_mode',
                                                   'camera_image')
        g_pool.display_mode_info_text = {
            'camera_image':
            "Raw eye camera image. This uses the least amount of CPU power",
            'roi':
            "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            'algorithm':
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."
        }

        g_pool.u_r = UIRoi(
            (g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get('roi')
        if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        writer = None

        pupil_detector_settings = session_settings.get(
            'pupil_detector_settings', None)
        last_pupil_detector = pupil_detectors[session_settings.get(
            'last_pupil_detector', Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,
                                                    pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)

        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s" % eye_id
        width, height = session_settings.get('window_size',
                                             g_pool.capture.frame_size)
        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        glfw.glfwSwapInterval(0)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale', 1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                           pos=(-300, 0),
                                           size=(0, 0),
                                           header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(
            ui.Slider('scale',
                      g_pool.gui,
                      setter=set_scale,
                      step=.05,
                      min=1.,
                      max=2.5,
                      label='Interface Size'))
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, frame.width, frame.height)))
        general_settings.append(
            ui.Switch('flip', g_pool, label='Flip image display'))
        general_settings.append(
            ui.Selector('display_mode',
                        g_pool,
                        setter=set_display_mode_info,
                        selection=['camera_image', 'roi', 'algorithm'],
                        labels=['Camera Image', 'ROI', 'Algorithm'],
                        label="Mode"))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector(
            'pupil_detector',
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Detector_2D, Detector_3D],
            labels=['C++ 2d detector', 'C++ 3d detector'],
            label="Detection method")
        general_settings.append(detector_selector)

        g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
        g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
        g_pool.capture.init_gui()

        g_pool.sidebar.append(general_settings)
        g_pool.sidebar.append(g_pool.capture_selector_menu)
        g_pool.sidebar.append(g_pool.capture_source_menu)

        g_pool.pupil_detector.init_gui(g_pool.sidebar)

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__: c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](
            g_pool, **manager_settings)
        g_pool.capture_manager.init_gui()

        def open_manager(manager_class):
            g_pool.capture_manager.cleanup()
            g_pool.capture_manager = manager_class(g_pool)
            g_pool.capture_manager.init_gui()

        #We add the capture selection menu, after a manager has been added:
        g_pool.capture_selector_menu.insert(
            0,
            ui.Selector('capture_manager',
                        g_pool,
                        setter=open_manager,
                        getter=lambda: g_pool.capture_manager.__class__,
                        selection=manager_classes,
                        labels=[b.gui_name for b in manager_classes],
                        label='Manager'))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_key)
        glfw.glfwSetCharCallback(main_window, on_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"

        should_publish_frames = False
        frame_publish_format = 'jpeg'

        #create a timer to control window update frequency
        window_update_timer = timer(1 / 60.)

        def window_should_update():
            return next(window_update_timer)

        logger.warning('Process started.')

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification['subject']
                if subject == 'eye_process.should_stop':
                    if notification['eye_id'] == eye_id:
                        break
                elif subject == 'set_detection_mapping_mode':
                    if notification['mode'] == '3d':
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    else:
                        if not isinstance(g_pool.pupil_detector, Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                elif subject == 'recording.started':
                    if notification['record_eye']:
                        record_path = notification['rec_path']
                        raw_mode = notification['compression']
                        logger.info("Will save eye video to: %s" % record_path)
                        timestamps_path = os.path.join(
                            record_path, "eye%s_timestamps.npy" % eye_id)
                        if raw_mode and frame.jpeg_buffer:
                            video_path = os.path.join(record_path,
                                                      "eye%s.mp4" % eye_id)
                            writer = JPEG_Writer(video_path,
                                                 g_pool.capture.frame_rate)
                        else:
                            video_path = os.path.join(record_path,
                                                      "eye%s.mp4" % eye_id)
                            writer = AV_Writer(video_path,
                                               g_pool.capture.frame_rate)
                        timestamps = []
                elif subject == 'recording.stopped':
                    if writer:
                        logger.info("Done recording.")
                        writer.release()
                        writer = None
                        np.save(timestamps_path, np.asarray(timestamps))
                        del timestamps
                elif subject.startswith('meta.should_doc'):
                    ipc_socket.notify({
                        'subject': 'meta.doc',
                        'actor': 'eye%i' % eye_id,
                        'doc': eye.__doc__
                    })
                elif subject.startswith('frame_publishing.started'):
                    should_publish_frames = True
                    frame_publish_format = notification.get('format', 'jpeg')
                elif subject.startswith('frame_publishing.stopped'):
                    should_publish_frames = False
                    frame_publish_format = 'jpeg'
                else:
                    g_pool.capture_manager.on_notify(notification)

            # Get an image from the grabber
            try:
                frame = g_pool.capture.get_frame()
            except StreamError as e:
                logger.error("Error getting frame. Stopping eye process.")
                logger.debug("Caught error: %s" % e)
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                g_pool.capture.seek_to_frame(0)
                frame = g_pool.capture.get_frame()

            g_pool.u_r = UIRoi((frame.height, frame.width))
            g_pool.capture_manager.update(frame, {})

            if should_publish_frames and frame.jpeg_buffer:
                if frame_publish_format == "jpeg":
                    data = frame.jpeg_buffer
                elif frame_publish_format == "yuv":
                    data = frame.yuv_buffer
                elif frame_publish_format == "bgr":
                    data = frame.bgr
                elif frame_publish_format == "gray":
                    data = frame.gray
                pupil_socket.send(
                    'frame.eye.%s' % eye_id, {
                        'width': frame.width,
                        'height': frame.width,
                        'index': frame.index,
                        'timestamp': frame.timestamp,
                        'format': frame_publish_format,
                        '__raw_data__': [data]
                    })

            #update performace graphs
            t = frame.timestamp
            dt, ts = t - ts, t
            try:
                fps_graph.add(1. / dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)

            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(
                frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            pupil_socket.send('pupil.%s' % eye_id, result)

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image', 'roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    window_size = glfw.glfwGetWindowSize(main_window)
                    make_coord_system_pixel_based(
                        (frame.height, frame.width, 3), g_pool.flip)
                    g_pool.capture.gl_display()

                    if result['method'] == '3d c++':

                        eye_ball = result['projected_sphere']
                        try:
                            pts = cv2.ellipse2Poly(
                                (int(eye_ball['center'][0]),
                                 int(eye_ball['center'][1])),
                                (int(eye_ball['axes'][0] / 2),
                                 int(eye_ball['axes'][1] / 2)),
                                int(eye_ball['angle']), 0, 360, 8)
                        except ValueError as e:
                            pass
                        else:
                            draw_polyline(
                                pts, 2,
                                RGBA(0., .9, .1, result['model_confidence']))

                    if result['confidence'] > 0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly(
                                (int(result['ellipse']['center'][0]),
                                 int(result['ellipse']['center'][1])),
                                (int(result['ellipse']['axes'][0] / 2),
                                 int(result['ellipse']['axes'][1] / 2)),
                                int(result['ellipse']['angle']), 0, 360, 15)
                            confidence = result[
                                'confidence'] * 0.7  #scale it a little
                            draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
                            draw_points([result['ellipse']['center']],
                                        size=20,
                                        color=RGBA(1., 0., 0., confidence),
                                        sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize(
                )  #detector decides if we visualize or not

        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path, np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings[
            'capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict(
            )
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = g_pool.version
        session_settings[
            'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.capture.deinit_gui()
        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()
        logger.info("Process shutting down.")
Ejemplo n.º 6
0
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url,ipc_push_url, user_dir, version, eye_id,overwrite_cap_settings=None):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """


    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools
    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=("notify",))

    with Is_Alive_Manager(is_alive_flag,ipc_socket,eye_id):

        #logging setup
        import logging
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logger = logging.getLogger()
        logger.handlers = []
        logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url))
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui,graph,cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
        import OpenGL.GL as gl
        from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based,is_window_visible
        from ui_roi import UIRoi
        #monitoring
        import psutil
        import math


        # helpers/utils
        from uvc import get_time_monotonic, StreamError
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from av_writer import JPEG_Writer,AV_Writer

        from video_capture import InitialisationError,StreamError, Fake_Source,EndofVideoFileError, source_classes, manager_classes
        source_by_name = {src.class_name():src for src in source_classes}

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}



        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600,300*eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600,31+300*eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600,300*eye_id)


        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic()-g_pool.timebase.value
        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window,w, h):
            if is_window_visible(window):
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w,h)
                graph.adjust_size(w,h)
                adjust_gl_view(w,h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key,scancode,action,mods)

        def on_char(window,char):
            g_pool.gui.update_char(char)

        def on_iconify(window,iconified):
            g_pool.iconified = iconified

        def on_button(window,button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1-pos[0],1-pos[1]
                    pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
                        return # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button,action,mods)



        def on_pos(window,x, y):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height) )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)

        def on_scroll(window,x,y):
            g_pool.gui.update_scroll(x,y*scroll_factor)

        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
        if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
            logger.info("Session setting are from older version of this app. I will not use those.")
            session_settings.clear()

        capture_manager_settings = session_settings.get(
            'capture_manager_settings', ('UVC_Manager',{}))

        if eye_id == 0:
            cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
        else:
            cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]

        # Initialize capture
        default_settings = {
            'source_class_name': 'UVC_Source',
            'preferred_names'  : cap_src,
            'frame_size': (640,480),
            'frame_rate': 90
        }
        settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
        try:
            cap = source_by_name[settings['source_class_name']](g_pool, **settings)
        except (KeyError,InitialisationError) as e:
            if isinstance(e,KeyError):
                logger.warning('Incompatible capture setting encountered. Falling back to fake source.')
            cap = Fake_Source(g_pool, **settings)

        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get('flip',False)
        g_pool.display_mode = session_settings.get('display_mode','camera_image')
        g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                    'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
                                    'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}



        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get('roi')
        if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        writer = None

        pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
        last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()


        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]


        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)


        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s"%eye_id
        width,height = session_settings.get('window_size',g_pool.capture.frame_size)
        main_window = glfw.glfwCreateWindow(width,height, title, None, None)
        window_pos = session_settings.get('window_position',window_position_default)
        glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        glfw.glfwSwapInterval(0)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale',1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
        general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
        general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
        general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
        g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
        general_settings.append(detector_selector)

        g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
        g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
        g_pool.capture.init_gui()

        g_pool.sidebar.append(general_settings)
        g_pool.sidebar.append(g_pool.capture_selector_menu)
        g_pool.sidebar.append(g_pool.capture_source_menu)

        g_pool.pupil_detector.init_gui(g_pool.sidebar)

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__:c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
        g_pool.capture_manager.init_gui()

        def open_manager(manager_class):
            g_pool.capture_manager.cleanup()
            g_pool.capture_manager = manager_class(g_pool)
            g_pool.capture_manager.init_gui()

        #We add the capture selection menu, after a manager has been added:
        g_pool.capture_selector_menu.insert(0,ui.Selector(
            'capture_manager',g_pool,
            setter    = open_manager,
            getter    = lambda: g_pool.capture_manager.__class__,
            selection = manager_classes,
            labels    = [b.gui_name for b in manager_classes],
            label     = 'Manager'
        ))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
        glfw.glfwSetKeyCallback(main_window,on_key)
        glfw.glfwSetCharCallback(main_window,on_char)
        glfw.glfwSetMouseButtonCallback(main_window,on_button)
        glfw.glfwSetCursorPosCallback(main_window,on_pos)
        glfw.glfwSetScrollCallback(main_window,on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))


        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config',{})


        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20,130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140,130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"

        should_publish_frames = False
        frame_publish_format = 'jpeg'

        #create a timer to control window update frequency
        window_update_timer = timer(1/60.)
        def window_should_update():
            return next(window_update_timer)

        logger.warning('Process started.')

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t,notification = notify_sub.recv()
                subject = notification['subject']
                if subject == 'eye_process.should_stop':
                    if notification['eye_id'] == eye_id:
                        break
                elif subject == 'set_detection_mapping_mode':
                    if notification['mode'] == '3d':
                        if not isinstance(g_pool.pupil_detector,Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only  = True
                    else:
                        if not isinstance(g_pool.pupil_detector,Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                elif subject == 'recording.started':
                    if notification['record_eye']:
                        record_path = notification['rec_path']
                        raw_mode = notification['compression']
                        logger.info("Will save eye video to: %s"%record_path)
                        timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
                        if raw_mode and frame.jpeg_buffer:
                            video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                            writer = JPEG_Writer(video_path,g_pool.capture.frame_rate)
                        else:
                            video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                            writer = AV_Writer(video_path,g_pool.capture.frame_rate)
                        timestamps = []
                elif subject == 'recording.stopped':
                    if writer:
                        logger.info("Done recording.")
                        writer.release()
                        writer = None
                        np.save(timestamps_path,np.asarray(timestamps))
                        del timestamps
                elif subject.startswith('meta.should_doc'):
                    ipc_socket.notify({
                        'subject':'meta.doc',
                        'actor':'eye%i'%eye_id,
                        'doc':eye.__doc__
                        })
                elif subject.startswith('frame_publishing.started'):
                    should_publish_frames = True
                    frame_publish_format = notification.get('format','jpeg')
                elif subject.startswith('frame_publishing.stopped'):
                    should_publish_frames = False
                    frame_publish_format = 'jpeg'
                else:
                    g_pool.capture_manager.on_notify(notification)

            # Get an image from the grabber
            try:
                frame = g_pool.capture.get_frame()
            except StreamError as e:
                logger.error("Error getting frame. Stopping eye process.")
                logger.debug("Caught error: %s"%e)
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                g_pool.capture.seek_to_frame(0)
                frame = g_pool.capture.get_frame()

            g_pool.u_r = UIRoi((frame.height,frame.width))
            g_pool.capture_manager.update(frame, {})

            if should_publish_frames and frame.jpeg_buffer:
                if   frame_publish_format == "jpeg":
                    data = frame.jpeg_buffer
                elif frame_publish_format == "yuv":
                    data = frame.yuv_buffer
                elif frame_publish_format == "bgr":
                    data = frame.bgr
                elif frame_publish_format == "gray":
                    data = frame.gray
                pupil_socket.send('frame.eye.%s'%eye_id,{
                    'width': frame.width,
                    'height': frame.width,
                    'index': frame.index,
                    'timestamp': frame.timestamp,
                    'format': frame_publish_format,
                    '__raw_data__': [data]
                })


            #update performace graphs
            t = frame.timestamp
            dt,ts = t-ts,t
            try:
                fps_graph.add(1./dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()



            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)


            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            pupil_socket.send('pupil.%s'%eye_id,result)

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image','roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    window_size =  glfw.glfwGetWindowSize(main_window)
                    make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
                    g_pool.capture.gl_display()

                    if result['method'] == '3d c++':

                        eye_ball = result['projected_sphere']
                        try:
                            pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
                                                (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
                                                int(eye_ball['angle']),0,360,8)
                        except ValueError as e:
                            pass
                        else:
                            draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )

                    if result['confidence'] >0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
                                            (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
                                            int(result['ellipse']['angle']),0,360,15)
                            confidence = result['confidence'] * 0.7 #scale it a little
                            draw_polyline(pts,1,RGBA(1.,0,0,confidence))
                            draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize() #detector decides if we visualize or not


        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path,np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window) #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = g_pool.version
        session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.capture.deinit_gui()
        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()
        logger.info("Process shutting down.")