Example #1
0
    def update(self,frame,recent_pupil_positions,events):
        img = frame.img
        self.img_shape = frame.img.shape

        if self.robust_detection.value:
            self.markers = detect_markers_robust(img,
                                                grid_size = 5,
                                                prev_markers=self.markers,
                                                min_marker_perimeter=self.min_marker_perimeter,
                                                aperture=self.aperture.value,
                                                visualize=0,
                                                true_detect_every_frame=3)
        else:
            self.markers = detect_markers_simple(img,
                                                grid_size = 5,
                                                min_marker_perimeter=self.min_marker_perimeter,
                                                aperture=self.aperture.value,
                                                visualize=0)

        # locate surfaces
        for s in self.surfaces:
            s.locate(self.markers)
            if s.detected:
                events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})

        if self.draw_markers.value:
            draw_markers(img,self.markers)

        # edit surfaces by user
        if self.surface_edit_mode:
            window = glfwGetCurrentContext()
            pos = glfwGetCursorPos(window)
            pos = normalize(pos,glfwGetWindowSize(window))
            pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels

            for s,v_idx in self.edit_surfaces:
                if s.detected:
                    pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True)
                    new_pos =  s.img_to_ref_surface(np.array(pos))
                    s.move_vertex(v_idx,new_pos)

        #map recent gaze onto detected surfaces used for pupil server
        for s in self.surfaces:
            if s.detected:
                s.gaze_on_srf = []
                for p in recent_pupil_positions:
                    if p['norm_pupil'] is not None:
                        gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze'])))
                        p['realtime gaze on '+s.name] = gp_on_s
                        s.gaze_on_srf.append(gp_on_s)


        #allow surfaces to open/close windows
        for s in self.surfaces:
            if s.window_should_close:
                s.close_window()
            if s.window_should_open:
                s.open_window()
Example #2
0
    def _checkFrame(self, img):
        img_size = img.shape[1], img.shape[0]

        # Check whole frame
        if not self._flag_check_roi:
            b0, b1, b2, b3 = 0, img_size[0], 0, img_size[1]

        # Check roi
        else:
            previous_fingertip_center = self._previous_finger_dict["screen_pos"]
            # Set up the boundary of the roi
            temp = img_size[0] / 16
            if self._predict_motion is not None:
                predict_center = (
                    previous_fingertip_center[0] + self._predict_motion[0],
                    previous_fingertip_center[1] + self._predict_motion[1],
                )
                b0 = predict_center[0] - temp * 0.5 - abs(self._predict_motion[0]) * 2
                b1 = predict_center[0] + temp * 0.5 + abs(self._predict_motion[0]) * 2
                b2 = predict_center[1] - temp * 0.8 - abs(self._predict_motion[1]) * 2
                b3 = predict_center[1] + temp * 2.0 + abs(self._predict_motion[1]) * 2
            else:
                predict_center = previous_fingertip_center
                b0 = predict_center[0] - temp * 0.5
                b1 = predict_center[0] + temp * 0.5
                b2 = predict_center[1] - temp * 0.8
                b3 = predict_center[1] + temp * 2.0

            b0 = 0 if b0 < 0 else int(b0)
            b1 = img_size[0] - 1 if b1 > img_size[0] - 1 else int(b1)
            b2 = 0 if b2 < 0 else int(b2)
            b3 = img_size[1] - 1 if b3 > img_size[1] - 1 else int(b3)
            col_slice = b0, b1
            row_slice = b2, b3
            img = img[slice(*row_slice), slice(*col_slice)]

        handmask = self.method.generateMask(img)
        handmask_smooth = self._smoothmask(handmask)
        f_dict = self._findFingertip(handmask_smooth, img_size, b0, b2)

        if f_dict is not None:
            norm_pos = normalize(f_dict["fingertip_center"], img_size, flip_y=True)
            norm_rect_points = [
                normalize(p, img_size, flip_y=True) for p in f_dict["rect_points"]
            ]
            return {
                "screen_pos": f_dict["fingertip_center"],
                "norm_pos": norm_pos,
                "norm_rect_points": norm_rect_points,
            }
        else:
            return None
Example #3
0
 def on_button(window,button, action, mods):
     g_pool.gui.update_button(button,action,mods)
     pos = glfwGetCursorPos(window)
     pos = normalize(pos,glfwGetWindowSize(window))
     pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
     for p in g_pool.plugins:
         p.on_click(pos,button,action)
Example #4
0
    def _map_monocular(self,p):
        if '3d' not in p['method']:
            return None

        gaze_point =  np.array(p['circle_3d']['normal'] ) * self.gaze_distance  + np.array( p['sphere']['center'] )

        image_point, _  =  cv2.projectPoints( np.array([gaze_point]) , self.rotation_vector, self.translation_vector , self.camera_matrix , self.dist_coefs )
        image_point = image_point.reshape(-1,2)
        image_point = normalize( image_point[0], self.world_frame_size , flip_y = True)

        eye_center = self.toWorld(p['sphere']['center'])
        gaze_3d = self.toWorld(gaze_point)
        normal_3d = np.dot( self.rotation_matrix, np.array( p['circle_3d']['normal'] ) )

        g = {   'norm_pos':image_point,
                'eye_center_3d':eye_center.tolist(),
                'gaze_normal_3d':normal_3d.tolist(),
                'gaze_point_3d':gaze_3d.tolist(),
                'confidence':p['confidence'],
                'timestamp':p['timestamp'],
                'base_data':[p]}

        if self.visualizer.window:
            self.gaze_pts_debug.append( gaze_3d )
            self.sphere['center'] = eye_center #eye camera coordinates
            self.sphere['radius'] = p['sphere']['radius']
        return g
    def update(self,frame,events):

        gaze_pts = []
        for p in events['pupil_positions']:
            if p['method'] == '3d c++' and p['confidence'] > self.g_pool.pupil_confidence_threshold:

                gaze_point =  np.array(p['circle_3d']['normal'] ) * self.gaze_distance  + np.array( p['sphere']['center'] )

                image_point, _  =  cv2.projectPoints( np.array([gaze_point]) , self.rotation_vector, self.translation_vector , self.camera_matrix , self.dist_coefs )
                image_point = image_point.reshape(-1,2)
                image_point = normalize( image_point[0], (frame.width, frame.height) , flip_y = True)

                eye_center = self.toWorld(p['sphere']['center'])
                gaze_3d = self.toWorld(gaze_point)
                normal_3d = np.dot( self.rotation_matrix, np.array( p['circle_3d']['normal'] ) )

                gaze_pts.append({   'norm_pos':image_point,
                                    'eye_center_3d':eye_center.tolist(),
                                    'gaze_normal_3d':normal_3d.tolist(),
                                    'gaze_point_3d':gaze_3d.tolist(),
                                    'confidence':p['confidence'],
                                    'timestamp':p['timestamp'],
                                    'base':[p]})

                if self.visualizer.window:
                    self.gaze_pts_debug.append( gaze_3d )
                    self.sphere['center'] = eye_center #eye camera coordinates
                    self.sphere['radius'] = p['sphere']['radius']

        events['gaze_positions'] = gaze_pts
Example #6
0
 def on_resize(window,w, h):
     active_window = glfwGetCurrentContext()
     glfwMakeContextCurrent(window)
     norm_size = normalize((w,h),glfwGetWindowSize(window))
     fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
     atb.TwWindowSize(*map(int,fb_size))
     glfwMakeContextCurrent(active_window)
    def update(self,frame,recent_pupil_positions,events):
        if self.active:
            img = frame.img
            if self.first_img is None:
                self.first_img = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)

            self.detected = False

            if self.count:
                gray = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
                # in cv2.3 nextPts is falsy required as an argument.
                nextPts_dummy = self.point.copy()
                nextPts,status, err = cv2.calcOpticalFlowPyrLK(self.first_img,gray,self.point,nextPts_dummy,winSize=(100,100))
                if status[0]:
                    self.detected = True
                    self.point = nextPts
                    self.first_img = gray
                    nextPts = nextPts[0]
                    self.pos = normalize(nextPts,(img.shape[1],img.shape[0]),flip_y=True)
                    self.count -=1

                    ref = {}
                    ref["norm_pos"] = self.pos
                    ref["timestamp"] = frame.timestamp
                    self.ref_list.append(ref)

            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['norm_pupil'] is not None:
                    self.pupil_list.append(p_pt)
Example #8
0
    def recent_events(self, events):
        frame = events.get("frame")
        if not frame:
            return
        if self.drag_offset is not None:
            pos = glfwGetCursorPos(glfwGetCurrentContext())
            pos = normalize(pos, glfwGetWindowSize(glfwGetCurrentContext()))
            pos = denormalize(
                pos, (frame.img.shape[1], frame.img.shape[0])
            )  # Position in img pixels
            self.pos[0] = pos[0] + self.drag_offset[0]
            self.pos[1] = pos[1] + self.drag_offset[1]

        if self.watermark is not None:
            # keep in image bounds, do this even when not dragging because the image sizes could change.
            self.pos[1] = max(
                0,
                min(frame.img.shape[0] - self.watermark.shape[0], max(self.pos[1], 0)),
            )
            self.pos[0] = max(
                0,
                min(frame.img.shape[1] - self.watermark.shape[1], max(self.pos[0], 0)),
            )
            pos = int(self.pos[0]), int(self.pos[1])
            img = frame.img
            roi = (
                slice(pos[1], pos[1] + self.watermark.shape[0]),
                slice(pos[0], pos[0] + self.watermark.shape[1]),
            )
            w_roi = slice(0, img.shape[0] - pos[1]), slice(0, img.shape[1] - pos[0])
            img[roi] = self.watermark[w_roi] * self.alpha_mask[w_roi] + img[roi] * (
                1 - self.alpha_mask[w_roi]
            )
Example #9
0
def get_markers_data(detection, img_size, timestamp):
    return {
        "id": detection.tag_id,
        "verts": detection.corners[::-1].tolist(),
        "centroid": normalize(detection.center, img_size, flip_y=True),
        "timestamp": timestamp,
    }
Example #10
0
 def uroi_on_mouse_button(button, action, mods):
     if g_pool.display_mode == "roi":
         if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
             g_pool.u_r.active_edit_pt = False
             # if the roi interacts we dont want
             # the gui to interact as well
             return
         elif action == glfw.GLFW_PRESS:
             x, y = glfw.glfwGetCursorPos(main_window)
             # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
             x *= hdpi_factor
             y *= hdpi_factor
             pos = normalize((x, y), camera_render_size)
             if g_pool.flip:
                 pos = 1 - pos[0], 1 - pos[1]
             # Position in img pixels
             pos = denormalize(
                 pos, g_pool.capture.frame_size
             )  # Position in img pixels
             if g_pool.u_r.mouse_over_edit_pt(
                 pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size
             ):
                 # if the roi interacts we dont want
                 # the gui to interact as well
                 return
Example #11
0
 def on_button(window,button, action, mods):
     if not atb.TwEventMouseButtonGLFW(button,action):
         pos = glfwGetCursorPos(window)
         pos = normalize(pos,glfwGetWindowSize(world_window))
         pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
         for p in g_pool.plugins:
             p.on_click(pos,button,action)
Example #12
0
 def on_button(button, pressed):
     if not atb.TwEventMouseButtonGLFW(button,pressed):
         if pressed:
             pos = glfwGetMousePos()
             pos = normalize(pos,glfwGetWindowSize())
             pos = denormalize(pos,(img.shape[1],img.shape[0]) ) #pos in img pixels
             ref.detector.new_ref(pos)
Example #13
0
    def update(self,frame,events):

        pupil_pts_0 = []
        pupil_pts_1 = []
        for p in events['pupil_positions']:
            if p['confidence'] > self.g_pool.pupil_confidence_threshold:
                if p['id'] == 0:
                    pupil_pts_0.append(p)
                else:
                    pupil_pts_1.append(p)

        # try binocular mapping (needs at least 1 pupil position in each list)
        gaze_pts = []
        if len(pupil_pts_0) > 0 and len(pupil_pts_1) > 0:
            gaze_pts = self.map_binocular_intersect(pupil_pts_0, pupil_pts_1 ,frame )
        # fallback to monocular if something went wrong
        else:
            for p in pupil_pts_0:

                gaze_point =  np.array(p['circle_3d']['normal'] ) * self.last_gaze_distance  + np.array( p['sphere']['center'] )

                image_point, _  =  cv2.projectPoints( np.array([gaze_point]) , self.rotation_vector0, self.translation_vector0 , self.camera_matrix , self.dist_coefs )
                image_point = image_point.reshape(-1,2)
                image_point = normalize( image_point[0], (frame.width, frame.height) , flip_y = True)
                gaze_pts.append({'norm_pos':image_point,'confidence':p['confidence'],'timestamp':p['timestamp'],'base':[p]})

                if self.visualizer.window:
                    self.gaze_pts_debug0.append( self.eye0_to_World(gaze_point) )
                    self.sphere0['center'] = self.eye0_to_World(p['sphere']['center']) #eye camera coordinates
                    self.sphere0['radius'] = p['sphere']['radius']


            for p in pupil_pts_1:

                gaze_point =  np.array(p['circle_3d']['normal'] ) * self.last_gaze_distance  + np.array( p['sphere']['center'] )

                image_point, _  =  cv2.projectPoints( np.array([gaze_point]) , self.rotation_vector1, self.translation_vector1 , self.camera_matrix , self.dist_coefs )
                image_point = image_point.reshape(-1,2)
                image_point = normalize( image_point[0], (frame.width, frame.height) , flip_y = True)
                gaze_pts.append({'norm_pos':image_point,'confidence':p['confidence'],'timestamp':p['timestamp'],'base':[p]})

                if self.visualizer.window:
                    self.gaze_pts_debug1.append( self.eye1_to_World(gaze_point) )
                    self.sphere1['center'] = self.eye1_to_World(p['sphere']['center']) #eye camera coordinates
                    self.sphere1['radius'] = p['sphere']['radius']

        events['gaze_positions'] = gaze_pts
Example #14
0
    def update(self, frame, events):
        img = frame.img
        img_shape = img.shape[:-1][::-1]  # width,height

        succeeding_frame = frame.index - self.prev_frame_idx == 1
        same_frame = frame.index == self.prev_frame_idx
        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # vars for calcOpticalFlowPyrLK
        lk_params = dict(
            winSize=(90, 90), maxLevel=3, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 20, 0.03)
        )

        updated_past_gaze = []

        # lets update past gaze using optical flow: this is like sticking the gaze points onto the pixels of the img.
        if self.past_gaze_positions and succeeding_frame:
            past_screen_gaze = np.array(
                [denormalize(ng["norm_pos"], img_shape, flip_y=True) for ng in self.past_gaze_positions],
                dtype=np.float32,
            )
            new_pts, status, err = cv2.calcOpticalFlowPyrLK(
                self.prev_gray, gray_img, past_screen_gaze, minEigThreshold=0.005, **lk_params
            )
            for gaze, new_gaze_pt, s, e in zip(self.past_gaze_positions, new_pts, status, err):
                if s:
                    # print "norm,updated",gaze['norm_gaze'], normalize(new_gaze_pt,img_shape[:-1],flip_y=True)
                    gaze["norm_pos"] = normalize(new_gaze_pt, img_shape, flip_y=True)
                    updated_past_gaze.append(gaze)
                    # logger.debug("updated gaze")

                else:
                    # logger.debug("dropping gaze")
                    # Since we will replace self.past_gaze_positions later,
                    # not appedning tu updated_past_gaze is like deliting this data point.
                    pass
        else:
            # we must be seeking, do not try to do optical flow, or pausing: see below.
            pass

        if same_frame:
            # paused
            # re-use last result
            events["gaze_positions"][:] = self.past_gaze_positions[:]
        else:
            # trim gaze that is too old
            if events["gaze_positions"]:
                now = events["gaze_positions"][0]["timestamp"]
                cutoff = now - self.timeframe
                updated_past_gaze = [g for g in updated_past_gaze if g["timestamp"] > cutoff]

            # inject the scan path gaze points into recent_gaze_positions
            events["gaze_positions"][:] = updated_past_gaze + events["gaze_positions"]
            events["gaze_positions"].sort(key=lambda x: x["timestamp"])  # this may be redundant...

        # update info for next frame.
        self.prev_gray = gray_img
        self.prev_frame_idx = frame.index
        self.past_gaze_positions = events["gaze_positions"]
    def update(self,frame,events):
        if self.active:
            recent_pupil_positions = events['pupil_positions']
            gray_img = frame.gray

            if self.clicks_to_close <=0:
                self.stop()
                return

            #detect the marker
            self.candidate_ellipses = get_candidate_ellipses(gray_img,
                                                            area_threshold=self.area_threshold,
                                                            dist_threshold=self.dist_threshold,
                                                            min_ring_count=5,
                                                            visual_debug=False)

            if len(self.candidate_ellipses) > 0:
                self.detected= True
                marker_pos = self.candidate_ellipses[0][0]
                self.pos = normalize(marker_pos,(frame.width,frame.height),flip_y=True)

            else:
                self.detected = False
                self.pos = None #indicate that no reference is detected


            #only save a valid ref position if within sample window of calibraiton routine
            on_position = self.lead_in < self.screen_marker_state < (self.lead_in+self.sample_duration)

            if on_position and self.detected:
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.g_pool.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            # Animate the screen marker
            if self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out:
                if self.detected or not on_position:
                    self.screen_marker_state += 1
            else:
                self.screen_marker_state = 0
                if not self.sites:
                    self.stop()
                    return
                self.active_site = self.sites.pop(0)
                logger.debug("Moving screen marker to site at %s %s"%tuple(self.active_site))



            #use np.arrays for per element wise math
            self.display_pos = np.array(self.active_site)
            self.on_position = on_position
            self.button.status_text = '%s / %s'%(self.active_site,9)
Example #16
0
    def update(self,frame,events):
        self.img_shape = frame.height,frame.width,3

        if self.running:
            gray = frame.gray

            if self.robust_detection:
                self.markers = detect_markers_robust(gray,
                                                    grid_size = 5,
                                                    prev_markers=self.markers,
                                                    min_marker_perimeter=self.min_marker_perimeter,
                                                    aperture=self.aperture,
                                                    visualize=0,
                                                    true_detect_every_frame=3,
                                                    invert_image=self.invert_image)
            else:
                self.markers = detect_markers(gray,
                                                grid_size = 5,
                                                min_marker_perimeter=self.min_marker_perimeter,
                                                aperture=self.aperture,
                                                visualize=0,
                                                invert_image=self.invert_image)


            if self.mode == "Show marker IDs":
                draw_markers(frame.gray,self.markers)

        events['surface'] = []

        # locate surfaces
        for s in self.surfaces:
            s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d)
            if s.detected:
                events['surface'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(), 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist()})

        if self.running:
            self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces))
        else:
            self.button.status_text = 'tracking paused'

        if self.mode == 'Show Markers and Surfaces':
            # edit surfaces by user
            if self.edit_surf_verts:
                window = glfwGetCurrentContext()
                pos = glfwGetCursorPos(window)
                pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
                for s,v_idx in self.edit_surf_verts:
                    if s.detected:
                        new_pos = s.img_to_ref_surface(np.array(pos))
                        s.move_vertex(v_idx,new_pos)

        #map recent gaze onto detected surfaces used for pupil server
        for s in self.surfaces:
            if s.detected:
                s.gaze_on_srf = []
                for p in events.get('gaze_positions',[]):
                    gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
                    p['realtime gaze on ' + s.name] = gp_on_s
                    s.gaze_on_srf.append(gp_on_s)
Example #17
0
 def on_button(window, button, action, mods):
     g_pool.gui.update_button(button, action, mods)
     pos = glfw.glfwGetCursorPos(window)
     pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
     # Position in img pixels
     pos = denormalize(pos, g_pool.capture.frame_size)
     for p in g_pool.plugins:
         p.on_click(pos, button, action)
Example #18
0
def current_mouse_pos(window, camera_render_size, frame_size):
    hdpi_fac = getHDPIFactor(window)
    x, y = glfwGetCursorPos(glfwGetCurrentContext())
    pos = x * hdpi_fac, y * hdpi_fac
    pos = normalize(pos, camera_render_size)
    # Position in img pixels
    pos = denormalize(pos, frame_size)
    return (int(pos[0]), int(pos[1]))
    def update(self,frame,recent_pupil_positions,events):
        self.img = frame.img
        self.img_shape = frame.img.shape
        self.update_marker_cache()
        self.markers = self.cache[frame.index]
        if self.markers == False:
            self.markers = []
            self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed

        # locate surfaces
        for s in self.surfaces:
            if not s.locate_from_cache(frame.index):
                s.locate(self.markers)
            if s.detected:
                events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})

        if self.mode.value == 4:
            draw_markers(frame.img,self.markers)

        # edit surfaces by user
        if self.mode.value == 1:
            window = glfwGetCurrentContext()
            pos = glfwGetCursorPos(window)
            pos = normalize(pos,glfwGetWindowSize(window))
            pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels

            for s,v_idx in self.edit_surfaces:
                if s.detected:
                    pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True)
                    new_pos =  s.img_to_ref_surface(np.array(pos))
                    s.move_vertex(v_idx,new_pos)
                    s.cache = None
                    self.heatmap = None
        else:
            # update srf with no or invald cache:
            for s in self.surfaces:
                if s.cache == None:
                    s.init_cache(self.cache)


        #allow surfaces to open/close windows
        for s in self.surfaces:
            if s.window_should_close:
                s.close_window()
            if s.window_should_open:
                s.open_window()
Example #20
0
 def on_button(button, pressed):
     if not atb.TwEventMouseButtonGLFW(button,pressed):
         if pressed:
             pos = glfwGetMousePos()
             pos = normalize(pos,glfwGetWindowSize())
             pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
             for p in g.plugins:
                 p.on_click(pos)
Example #21
0
 def gaze_mapper(self, pupil_positions):
     gaze_pts = []
     pupil_confidence_threshold = 0.6
     for p in pupil_positions:
         if p['confidence'] > self.g_pool.pupil_confidence_threshold:
             norm_pos = normalize()
             gaze_point = self.map_fn(p['norm_pos'])
             gaze_pts.append({'norm_pos':gaze_point,'confidence':p['confidence'],'timestamp':p['timestamp'],'base':[p]})
    def recent_events(self, events):
        frame = events.get('frame')
        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']
            gray_img = frame.gray

            if self.clicks_to_close <=0:
                self.stop()
                return

            # detect the marker
            self.markers = find_concetric_circles(gray_img, min_ring_count=4)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][0]  # first marker, innermost ellipse,center
                self.pos = normalize(marker_pos, (frame.width, frame.height), flip_y=True)

            else:
                self.detected = False
                self.pos = None  # indicate that no reference is detected

            # only save a valid ref position if within sample window of calibraiton routine
            on_position = self.lead_in < self.screen_marker_state < (self.lead_in+self.sample_duration)

            if on_position and self.detected:
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            # always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if on_position and self.detected and events.get('fixations', []):
                self.screen_marker_state = min(
                    self.sample_duration+self.lead_in,
                    self.screen_marker_state+self.fixation_boost)

            # Animate the screen marker
            if self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out:
                if self.detected or not on_position:
                    self.screen_marker_state += 1
            else:
                self.screen_marker_state = 0
                if not self.sites:
                    self.stop()
                    return
                self.active_site = self.sites.pop(0)
                logger.debug("Moving screen marker to site at {} {}".format(*self.active_site))

            # use np.arrays for per element wise math
            self.display_pos = np.array(self.active_site)
            self.on_position = on_position
            self.button.status_text = '{} / {}'.format(self.active_site, 9)
Example #23
0
    def update(self, frame, events):
        self.img_shape = frame.height, frame.width, 3

        if self.running:
            gray = frame.gray

            if self.robust_detection:
                self.markers = detect_markers_robust(
                    gray,
                    grid_size=5,
                    prev_markers=self.markers,
                    min_marker_perimeter=self.min_marker_perimeter,
                    aperture=self.aperture,
                    visualize=0,
                    true_detect_every_frame=3,
                )
            else:
                self.markers = detect_markers(
                    gray,
                    grid_size=5,
                    min_marker_perimeter=self.min_marker_perimeter,
                    aperture=self.aperture,
                    visualize=0,
                )

            if self.mode == "Show marker IDs":
                draw_markers(frame.img, self.markers)

        # locate surfaces
        for s in self.surfaces:
            s.locate(self.markers, self.locate_3d, self.camera_intrinsics)
            # if s.detected:
            # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})

        if self.running:
            self.button.status_text = "%s/%s" % (len([s for s in self.surfaces if s.detected]), len(self.surfaces))
        else:
            self.button.status_text = "tracking paused"

        # edit surfaces by user
        if self.mode == "Surface edit mode":
            window = glfwGetCurrentContext()
            pos = glfwGetCursorPos(window)
            pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
            for s, v_idx in self.edit_surfaces:
                if s.detected:
                    new_pos = s.img_to_ref_surface(np.array(pos))
                    s.move_vertex(v_idx, new_pos)

        # map recent gaze onto detected surfaces used for pupil server
        for s in self.surfaces:
            if s.detected:
                s.gaze_on_srf = []
                for p in events.get("gaze_positions", []):
                    gp_on_s = tuple(s.img_to_ref_surface(np.array(p["norm_pos"])))
                    p["realtime gaze on " + s.name] = gp_on_s
                    s.gaze_on_srf.append(gp_on_s)
Example #24
0
 def on_pos(window, x, y):
     x, y = x * hdpi_factor, y * hdpi_factor
     g_pool.gui.update_mouse(x, y)
     pos = x, y
     pos = normalize(pos, g_pool.camera_render_size)
     # Position in img pixels
     pos = denormalize(pos, g_pool.capture.frame_size)
     for p in g_pool.plugins:
         p.on_pos(pos)
    def update(self,frame,events):
        self.img_shape = frame.img.shape
        self.update_marker_cache()
        # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
        self.markers = self.cache[frame.index]
        if self.markers == False:
            self.markers = []
            self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed


        events['surfaces'] = []
        # locate surfaces
        for s in self.surfaces:
            if not s.locate_from_cache(frame.index):
                s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter)
            if s.detected:
                events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})

        if self.mode == "Show marker IDs":
            draw_markers(frame.img,self.markers)

        elif self.mode == "Show Markers and Surfaces":
            # edit surfaces by user
            if self.edit_surf_verts:
                window = glfwGetCurrentContext()
                pos = glfwGetCursorPos(window)
                pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
                for s,v_idx in self.edit_surf_verts:
                    if s.detected:
                        new_pos =  s.img_to_ref_surface(np.array(pos))
                        s.move_vertex(v_idx,new_pos)
            else:
                # update srf with no or invald cache:
                for s in self.surfaces:
                    if s.cache == None and s not in [s for s,i in self.edit_surf_verts]:
                        s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter)
                        self.notify_all_delayed({'subject':'surfaces_changed'})



        #map recent gaze onto detected surfaces used for pupil server
        for s in self.surfaces:
            if s.detected:
                s.gaze_on_srf = []
                for p in events.get('gaze_positions',[]):
                    gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
                    p['realtime gaze on ' + s.name] = gp_on_s
                    s.gaze_on_srf.append(gp_on_s)


        #allow surfaces to open/close windows
        for s in self.surfaces:
            if s.window_should_close:
                s.close_window()
            if s.window_should_open:
                s.open_window()
Example #26
0
 def on_resize(window,w, h):
     active_window = glfwGetCurrentContext()
     glfwMakeContextCurrent(window)
     norm_size = normalize((w,h),glfwGetWindowSize(window))
     fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
     atb.TwWindowSize(*map(int,fb_size))
     adjust_gl_view(w,h,window)
     glfwMakeContextCurrent(active_window)
     for p in g_pool.plugins:
         p.on_window_resize(window,w,h)
Example #27
0
        def on_pos(window,x, y):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height) )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
Example #28
0
    def update(self,frame,events):
        self.img = frame.img
        self.img_shape = frame.img.shape
        self.update_marker_cache()
        self.markers = self.cache[frame.index]
        if self.markers == False:
            self.markers = []
            self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed

        # locate surfaces
        for s in self.surfaces:
            if not s.locate_from_cache(frame.index):
                s.locate(self.markers)
            if s.detected:
                pass
                # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})

        if self.mode == "Show marker IDs":
            draw_markers(frame.img,self.markers)

        # edit surfaces by user
        if self.mode == "Surface edit mode":
            window = glfwGetCurrentContext()
            pos = glfwGetCursorPos(window)
            pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)

            for s,v_idx in self.edit_surfaces:
                if s.detected:
                    new_pos =  s.img_to_ref_surface(np.array(pos))
                    s.move_vertex(v_idx,new_pos)
                    s.cache = None
                    self.heatmap = None
        else:
            # update srf with no or invald cache:
            for s in self.surfaces:
                if s.cache == None:
                    s.init_cache(self.cache)


        #map recent gaze onto detected surfaces used for pupil server
        for s in self.surfaces:
            if s.detected:
                s.gaze_on_srf = []
                for p in events.get('gaze_positions',[]):
                    gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
                    p['realtime gaze on ' + s.name] = gp_on_s
                    s.gaze_on_srf.append(gp_on_s)


        #allow surfaces to open/close windows
        for s in self.surfaces:
            if s.window_should_close:
                s.close_window()
            if s.window_should_open:
                s.open_window()
Example #29
0
        def on_pos(window, x, y):
            x *= hdpi_factor
            y *= hdpi_factor
            g_pool.gui.update_mouse(x, y)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, g_pool.capture.frame_size)
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
Example #30
0
        def on_pos(window, x, y):
            hdpi_factor = glfw.glfwGetFramebufferSize(
                window)[0] / glfw.glfwGetWindowSize(window)[0]
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,g_pool.capture.frame_size )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
Example #31
0
    def visualize(self, frame, alpha, scale, show_ellipses, pupil_positions):
        if not self.initialized:
            return

        requested_eye_frame_idx = self.eye_world_frame_map[frame.index]
        # 1. do we need a new frame?
        if requested_eye_frame_idx != self.current_eye_frame.index:
            if requested_eye_frame_idx == self.source.get_frame_index() + 2:
                # if we just need to seek by one frame, its faster to just read one and and throw it away.
                self.source.get_frame()
            if requested_eye_frame_idx != self.source.get_frame_index() + 1:
                self.source.seek_to_frame(int(requested_eye_frame_idx))

            try:
                self.current_eye_frame = self.source.get_frame()
            except EndofVideoError:
                logger.info(
                    "Reached the end of the eye video for eye video {}.".
                    format(self.eyeid))

        # 2. dragging image
        if self.drag_offset is not None:
            x, y = glfwGetCursorPos(glfwGetCurrentContext())
            pos = x * self.hdpi_fac, y * self.hdpi_fac
            pos = normalize(pos, self.g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))
            self.pos = int(pos[0] +
                           self.drag_offset[0]), int(pos[1] +
                                                     self.drag_offset[1])

        # 3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
        video_size = round(self.current_eye_frame.width * scale), round(
            self.current_eye_frame.height * scale)

        # frame.img.shape[0] is height, frame.img.shape[1] is width of screen
        self.pos = (min(frame.img.shape[1] - video_size[0],
                        max(self.pos[0], 0)),
                    min(frame.img.shape[0] - video_size[1],
                        max(self.pos[1], 0)))

        # 4. vflipping images, converting to greyscale
        eyeimage = self.current_eye_frame.gray
        eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)

        if show_ellipses:
            try:
                pp = next(
                    (pp for pp in pupil_positions if pp['id'] == self.eyeid
                     and pp['timestamp'] == self.current_eye_frame.timestamp))
            except StopIteration:
                pass
            else:
                el = pp['ellipse']
                conf = int(
                    pp.get('model_confidence', pp.get('confidence', 0.1)) *
                    255)
                el_points = getEllipsePts(
                    (el['center'], el["axes"], el['angle']))
                cv2.polylines(eyeimage, [np.asarray(el_points, dtype='i')],
                              True, (0, 0, 255, conf),
                              thickness=1)
                cv2.circle(eyeimage,
                           (int(el['center'][0]), int(el['center'][1])),
                           5, (0, 0, 255, conf),
                           thickness=-1)

        #flip and scale
        eyeimage = cv2.resize(eyeimage, (0, 0), fx=scale, fy=scale)
        if self.hflip:
            eyeimage = np.fliplr(eyeimage)
        if self.vflip:
            eyeimage = np.flipud(eyeimage)

        transparent_image_overlay(self.pos, eyeimage, frame.img, alpha)
Example #32
0
    def _check_frame(self, img):
        """
        Track the markers in the ROIs / in the whole frame

        :param img: input gray image
        :type img: numpy.ndarray
        :return: all detected markers including the information about their ellipses, center positions and their type
        (Ref/Stop)
        :rtype: a list containing dictionary with keys: 'ellipses', 'img_pos', 'norm_pos', 'marker_type'
        """
        img_size = img.shape[::-1]
        scale = 0.5 if img_size[0] >= 1280 else 640 / img_size[0]

        marker_list = []
        # Check whole frame
        if not self._flag_check_roi:
            ellipses_list = find_pupil_circle_marker(img, scale)

            # Save the markers in dictionaries
            for ellipses_ in ellipses_list:
                ellipses = ellipses_["ellipses"]
                img_pos = ellipses[0][0]
                norm_pos = normalize(img_pos, img_size, flip_y=True)
                marker_list.append(
                    {
                        "ellipses": ellipses,
                        "img_pos": img_pos,
                        "norm_pos": norm_pos,
                        "marker_type": ellipses_["marker_type"],
                    }
                )

        # Check roi
        else:
            for i in range(len(self._previous_markers)):
                largest_ellipse = self._previous_markers[i]["ellipses"][-1]

                # Set up the boundary of the roi
                if self._predict_motion:
                    predict_center = (
                        largest_ellipse[0][0] + self._predict_motion[i][0],
                        largest_ellipse[0][1] + self._predict_motion[i][1],
                    )
                    b0 = (
                        predict_center[0]
                        - largest_ellipse[1][1]
                        - abs(self._predict_motion[i][0]) * 2
                    )
                    b1 = (
                        predict_center[0]
                        + largest_ellipse[1][1]
                        + abs(self._predict_motion[i][0]) * 2
                    )
                    b2 = (
                        predict_center[1]
                        - largest_ellipse[1][0]
                        - abs(self._predict_motion[i][1]) * 2
                    )
                    b3 = (
                        predict_center[1]
                        + largest_ellipse[1][0]
                        + abs(self._predict_motion[i][1]) * 2
                    )
                else:
                    predict_center = largest_ellipse[0]
                    b0 = predict_center[0] - largest_ellipse[1][1]
                    b1 = predict_center[0] + largest_ellipse[1][1]
                    b2 = predict_center[1] - largest_ellipse[1][0]
                    b3 = predict_center[1] + largest_ellipse[1][0]

                b0 = 0 if b0 < 0 else int(b0)
                b1 = img_size[0] - 1 if b1 > img_size[0] - 1 else int(b1)
                b2 = 0 if b2 < 0 else int(b2)
                b3 = img_size[1] - 1 if b3 > img_size[1] - 1 else int(b3)
                col_slice = b0, b1
                row_slice = b2, b3

                ellipses_list = find_pupil_circle_marker(
                    img[slice(*row_slice), slice(*col_slice)], scale
                )

                # Track the marker which was detected last frame;
                # To avoid more than one markers are detected in one ROI
                if len(ellipses_list):
                    if len(ellipses_list) == 1:
                        right_ellipses = ellipses_list[0]
                    else:
                        pre_pos = np.array(
                            (
                                self._previous_markers[i]["img_pos"][0] - b0,
                                self._previous_markers[i]["img_pos"][1] - b2,
                            )
                        )
                        temp_dist = [
                            LA.norm(e["ellipses"][0][0] - pre_pos)
                            for e in ellipses_list
                        ]
                        right_ellipses = ellipses_list[temp_dist.index(min(temp_dist))]
                    ellipses = [
                        ((e[0][0] + b0, e[0][1] + b2), e[1], e[2])
                        for e in right_ellipses["ellipses"]
                    ]
                    img_pos = ellipses[0][0]
                    norm_pos = normalize(img_pos, img_size, flip_y=True)
                    # Save the marker in dictionary
                    marker_list.append(
                        {
                            "ellipses": ellipses,
                            "img_pos": img_pos,
                            "norm_pos": norm_pos,
                            "marker_type": right_ellipses["marker_type"],
                        }
                    )

        return marker_list
Example #33
0
    def recent_events(self, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        frame = events.get('frame')
        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']

            gray_img  = frame.gray

            if self.world_size is None:
                self.world_size = frame.width,frame.height

            self.markers = find_concetric_circles(gray_img,min_ring_count=3)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][0] #first marker innermost ellipse, pos
                self.pos = normalize(marker_pos,(frame.width,frame.height),flip_y=True)

            else:
                self.detected = False
                self.pos = None  # indicate that no reference is detected

            # center dark or white?
            if self.detected:
                second_ellipse = self.markers[0][1]
                col_slice = int(second_ellipse[0][0]-second_ellipse[1][0]/2),int(second_ellipse[0][0]+second_ellipse[1][0]/2)
                row_slice = int(second_ellipse[0][1]-second_ellipse[1][1]/2),int(second_ellipse[0][1]+second_ellipse[1][1]/2)
                marker_gray = gray_img[slice(*row_slice),slice(*col_slice)]
                avg = cv2.mean(marker_gray)[0]
                center = marker_gray[int(second_ellipse[1][1])//2, int(second_ellipse[1][0])//2]
                rel_shade = center-avg

                # auto_stop logic
                if rel_shade > 30:
                    # bright marker center found
                    self.auto_stop +=1
                    self.stop_marker_found = True

                else:
                    self.auto_stop = 0
                    self.stop_marker_found = False


            #tracking logic
            if self.detected and not self.stop_marker_found:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother*(pos-smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0])+abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother*(new_vel-self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos-np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0])+abs(sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if self.counter <= 0:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug("Steady marker found. Starting to sample {} datapoints".format(self.counter_max))
                        self.notify_all({'subject':'calibration.marker_found','timestamp':self.g_pool.get_timestamp(),'record':True})
                        self.counter = self.counter_max

                if self.counter > 0:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning("Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again.".format(self.counter_max-self.counter))
                        self.notify_all({'subject':'calibration.marker_moved_too_quickly','timestamp':self.g_pool.get_timestamp(),'record':True})
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = marker_pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if events.get('fixations', []):
                            self.counter -= self.fixation_boost
                        if self.counter <= 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug("Sampled {} datapoints. Stopping to sample. Looking for steady marker again.".format(self.counter_max))
                            self.notify_all({'subject':'calibration.marker_sample_completed','timestamp':self.g_pool.get_timestamp(),'record':True})


            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'



            #stop if autostop condition is satisfied:
            if self.auto_stop >=self.auto_stop_max:
                self.auto_stop = 0
                self.stop()


        else:
            pass
Example #34
0
        for index, d in enumerate(unbiased_gaze):
            bias = indexed_bias[index]
            f = d['frame']
            i = d['i']
            bias_by_frame[f][i] = bias

        # we must have at least the first gaze to continue no matter what happend before this line
        if len(bias_by_frame[0][0]) < 2:
            bias_by_frame[0][0] = bias_along_blocks[0]['bias']

        # normalize and reverse homographic transformation
        x_size, y_size = s.real_world_size['x'], s.real_world_size['y']
        for f in range(len(self.g_pool.gaze_positions_by_frame)):
            for i in range(len(self.g_pool.gaze_positions_by_frame[f])):
                if len(bias_by_frame[f][i]) == 2:
                    bias_by_frame[f][i] = normalize(bias_by_frame[f][i], (x_size,y_size), True)
                    #bias_by_frame[f][i] = s.ref_surface_to_img(np.array(bias_by_frame[f][i]))
                    
                    #m_to_screen = locate_m_to_screen(s, f)
                    #bias_by_frame[f][i] = surface_to_screen(np.array(bias_by_frame[f][i]), m_to_screen)
                    #bias_by_frame[f][i] = bias_by_frame[f][i]
                    last_bias = bias_by_frame[f][i]
                else:
                    # finally, we apply the bias correction to all gaze, not only to the filtered ones
                    bias_by_frame[f][i] = last_bias
        self.bias_by_frame = bias_by_frame
        return True

    def _set_gaze_correction(self):
        def apply_correction(f, i):
            x_bias, y_bias = self.bias_by_frame[f][i]
Example #35
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime

    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        # display
        import glfw

        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils

        # capture
        from video_capture import File_Source

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation

        # from vis_scan_path import Vis_Scan_Path
        from seek_control import Seek_Control
        from surface_tracker import Surface_Tracker_Offline

        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from eye_movement import Offline_Eye_Movement_Detector
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producer.gaze_from_recording import GazeFromRecording
        from gaze_producer.gaze_from_offline_calibration import (
            GazeFromOfflineCalibration, )
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from video_export.plugins.imotions_exporter import iMotions_Exporter
        from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
        from video_export.plugins.world_video_exporter import World_Video_Exporter
        from head_pose_tracker.offline_head_pose_tracker import Offline_Head_Pose_Tracker
        from video_capture import File_Source
        from video_overlay.plugins import Video_Overlay, Eye_Overlay

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.24"), "pyglui out of date, please upgrade to newest version"

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, "plugins"))
        system_plugins = [
            Log_Display,
            Seek_Control,
            Plugin_Manager,
            System_Graphs,
            System_Timelines,
            Audio_Playback,
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Eye_Overlay,
            Video_Overlay,
            # Vis_Scan_Path,
            Offline_Fixation_Detector,
            Offline_Eye_Movement_Detector,
            Offline_Blink_Detection,
            Surface_Tracker_Offline,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            GazeFromRecording,
            GazeFromOfflineCalibration,
            World_Video_Exporter,
            iMotions_Exporter,
            Eye_Video_Exporter,
            Offline_Head_Pose_Tracker,
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for path in paths:
                if pm.is_pupil_rec_dir(path):
                    _restart_with_recording(path)
                    return
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        def _restart_with_recording(rec_dir):
            logger.debug("Starting new session with '{}'".format(rec_dir))
            ipc_pub.notify({
                "subject": "player_drop_process.should_start",
                "rec_dir": rec_dir
            })
            glfw.glfwSetWindowShouldClose(g_pool.main_window, True)

        tick = delta_t()

        def get_dt():
            return next(tick)

        meta_info = pm.load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info("Application Version: {}".format(app_version))
        logger.info("System Info: {}".format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = SimpleNamespace()
        g_pool.app = "player"
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in valid_ext
        ][0]
        File_Source(
            g_pool,
            timing="external",
            source_path=video_path,
            buffered_decoding=True,
            fill_gaps=True,
        )

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        window_pos = session_settings.get("window_position",
                                          window_position_default)
        window_name = "Pupil Player: {} - {}".format(
            meta_info["Recording Name"],
            os.path.split(rec_dir)[-1])

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, window_name, None,
                                            None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.binocular = meta_info.get("Eye Mode",
                                         "monocular") == "binocular"
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.0
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT)
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT)

        # populated by producers
        g_pool.pupil_positions = pm.Bisector()
        g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()
        g_pool.eye_movements = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {"subject": "min_data_confidence_changed"}
            notification["_notify_time_"] = time() + 0.8
            g_pool.ipc_pub.notify(notification)

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop
            export_ts_window = pm.exact_window(g_pool.timestamps,
                                               (left_idx, right_idx))

            export_dir = os.path.join(g_pool.rec_dir, "exports")
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                "Player Software Version":
                str(g_pool.version),
                "Data Format Version":
                meta_info["Data Format Version"],
                "Export Date":
                strftime("%d.%m.%Y", localtime()),
                "Export Time":
                strftime("%H:%M:%S", localtime()),
                "Frame Index Range:":
                g_pool.seek_control.get_frame_index_trim_range_string(),
                "Relative Time Range":
                g_pool.seek_control.get_rel_time_trim_range_string(),
                "Absolute Time Range":
                g_pool.seek_control.get_abs_time_trim_range_string(),
            }
            with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
                write_key_value_file(csv, export_info)

            notification = {
                "subject": "should_export",
                "range": export_range,
                "ts_window": export_ts_window,
                "export_dir": export_dir,
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({
                "subject": "player_process.should_start",
                "rec_dir": rec_dir,
                "delay": 2.0,
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu("User Timelines",
                                                 pos=(0.0, -150.0),
                                                 size=(0.0, 0.0),
                                                 header_pos="headline")
        g_pool.user_timelines.color = RGBA(a=0.0)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2] +
                list(np.arange(1.5, 5.1, 0.5)),
                label="Interface Size",
            ))
        general_settings.append(
            ui.Info_Text("Player Version: {}".format(g_pool.version)))
        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(
                meta_info["Capture Software Version"])))
        general_settings.append(
            ui.Info_Text("Data Format Version: {}".format(
                meta_info["Data Format Version"])))

        general_settings.append(
            ui.Info_Text(
                "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
            ))
        general_settings.append(
            ui.Slider(
                "min_data_confidence",
                g_pool,
                setter=set_data_confidence,
                step=0.05,
                min=0.0,
                max=1.0,
                label="Minimum data confidence",
            ))

        general_settings.append(
            ui.Button("Restart with default settings", reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb(
            "export",
            label=chr(0xE2C5),
            getter=lambda: False,
            setter=do_export,
            hotkey="e",
            label_font="pupil_icons",
        )
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [
            ("Plugin_Manager", {}),
            ("Seek_Control", {}),
            ("Log_Display", {}),
            ("Raw_Data_Exporter", {}),
            ("Vis_Polyline", {}),
            ("Vis_Circle", {}),
            ("System_Graphs", {}),
            ("System_Timelines", {}),
            ("World_Video_Exporter", {}),
            ("Pupil_From_Recording", {}),
            ("GazeFromRecording", {}),
            ("Audio_Playback", {}),
        ]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins))

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                "rel_time_trim_section",
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label="Relative time range to export",
            ),
        )
        general_settings.insert(
            -1,
            ui.Text_Input(
                "frame_idx_trim_section",
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label="Frame index range to export",
            ),
        )

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n["subject"]
            if subject == "start_plugin":
                g_pool.plugins.add(g_pool.plugin_by_name[n["name"]],
                                   args=n.get("args", {}))
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify({
                    "subject": "meta.doc",
                    "actor": g_pool.app,
                    "doc": player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            "subject": "meta.doc",
                            "actor": p.class_name,
                            "doc": p.on_notify.__doc__,
                        })

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events["pupil"] = []
            events["gaze"] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            glfw.glfwPollEvents()
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                # present frames at appropriate speed
                g_pool.seek_control.wait(events["frame"].timestamp)
                glfw.glfwSwapBuffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["min_data_confidence"] = g_pool.min_data_confidence
        session_settings[
            "min_calibration_confidence"] = g_pool.min_calibration_confidence
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["window_position"] = glfw.glfwGetWindowPos(
            main_window)
        session_settings["version"] = str(g_pool.version)

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Player crashed with trace:\n{}".format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "player_process.stopped"})
        sleep(1.0)
Example #36
0
def circle_detector(ipc_push_url, pair_url,
                    source_path, timestamps_path, batch_size=20):

    # ipc setup
    import zmq
    import zmq_tools
    zmq_ctx = zmq.Context()
    process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url)

    # logging setup
    import logging
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    # imports
    import cv2
    import numpy as np
    from time import sleep
    from circle_detector import find_concetric_circles
    from video_capture import File_Source, EndofVideoFileError
    from methods import normalize

    try:
        src = File_Source(Empty(), source_path, np.load(timestamps_path), timed_playback=False)
        frame = src.get_frame()
        logger.info('Starting calibration marker detection...')
        frame_count = src.get_frame_count()

        queue = []

        while True:
            while process_pipe.new_data:
                topic, n = process_pipe.recv()
                if topic == 'terminate':
                    process_pipe.send(topic='exception', payload={"reason": "User terminated."})
                    logger.debug("Process terminated")
                    sleep(1.0)
                    return

            progress = 100.*frame.index/frame_count

            markers = find_concetric_circles(frame.gray, min_ring_count=3)
            if len(markers) > 0:
                detected = True
                marker_pos = markers[0][0][0]  # first marker innermost ellipse, pos
                pos = normalize(marker_pos, (frame.width, frame.height), flip_y=True)

            else:
                detected = False
                pos = None

            if detected:
                second_ellipse = markers[0][1]
                col_slice = int(second_ellipse[0][0]-second_ellipse[1][0]/2), int(second_ellipse[0][0]+second_ellipse[1][0]/2)
                row_slice = int(second_ellipse[0][1]-second_ellipse[1][1]/2), int(second_ellipse[0][1]+second_ellipse[1][1]/2)
                marker_gray = frame.gray[slice(*row_slice), slice(*col_slice)]
                avg = cv2.mean(marker_gray)[0]
                center = marker_gray[int(second_ellipse[1][1])//2, int(second_ellipse[1][0])//2]
                rel_shade = center-avg

                ref = {}
                ref["norm_pos"] = pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                ref['index'] = frame.index
                if rel_shade > 30:
                    ref['type'] = 'stop_marker'
                else:
                    ref['type'] = 'calibration_marker'

                queue.append((progress, ref))
            else:
                queue.append((progress, None))

            if len(queue) > batch_size:
                # dequeue batch
                data = queue[:batch_size]
                del queue[:batch_size]
                process_pipe.send(topic='progress', payload={'data': data})

            frame = src.get_frame()

    except EndofVideoFileError:
        process_pipe.send(topic='progress', payload={'data': queue})
        process_pipe.send(topic='finished', payload={})
        logger.debug("Process finished")

    except:
        import traceback
        process_pipe.send(topic='exception', payload={'reason': traceback.format_exc()})
        logger.debug("Process raised Exception")

    sleep(1.0)
Example #37
0
    def _map_binocular(self, p0, p1):

        if "3d" not in p0["method"] or "3d" not in p1["method"]:
            return None

        # find the nearest intersection point of the two gaze lines
        # eye ball centers in world coords
        s0_center = self.eye0_to_World(np.array(p0["sphere"]["center"]))
        s1_center = self.eye1_to_World(np.array(p1["sphere"]["center"]))
        # eye line of sight in world coords
        s0_normal = np.dot(
            self.rotation_matricies[0], np.array(p0["circle_3d"]["normal"])
        )
        s1_normal = np.dot(
            self.rotation_matricies[1], np.array(p1["circle_3d"]["normal"])
        )

        # See Lech Swirski: "Gaze estimation on glasses-based stereoscopic displays"
        # Chapter: 7.4.2 Cyclopean gaze estimate

        # the cyclop is the avg of both lines of sight
        cyclop_normal = (s0_normal + s1_normal) / 2.0
        cyclop_center = (s0_center + s1_center) / 2.0

        # We use it to define a viewing plane.
        gaze_plane = np.cross(cyclop_normal, s1_center - s0_center)
        gaze_plane = gaze_plane / np.linalg.norm(gaze_plane)

        # project lines of sight onto the gaze plane
        s0_norm_on_plane = s0_normal - np.dot(gaze_plane, s0_normal) * gaze_plane
        s1_norm_on_plane = s1_normal - np.dot(gaze_plane, s1_normal) * gaze_plane

        # create gaze lines on this plane
        gaze_line0 = [s0_center, s0_center + s0_norm_on_plane]
        gaze_line1 = [s1_center, s1_center + s1_norm_on_plane]

        # find the intersection of left and right line of sight.
        (
            nearest_intersection_point,
            intersection_distance,
        ) = math_helper.nearest_intersection(gaze_line0, gaze_line1)
        if nearest_intersection_point is not None and self.backproject:
            cyclop_gaze = nearest_intersection_point - cyclop_center
            self.last_gaze_distance = np.sqrt(cyclop_gaze.dot(cyclop_gaze))
            image_point = self.g_pool.capture.intrinsics.projectPoints(
                np.array([nearest_intersection_point])
            )
            image_point = image_point.reshape(-1, 2)
            image_point = normalize(
                image_point[0], self.g_pool.capture.intrinsics.resolution, flip_y=True
            )
            image_point = _clamp_norm_point(image_point)

        if hasattr(self, "visualizer") and self.visualizer.window:
            gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
            gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
            self.gaze_pts_debug0.append(gaze0_3d)
            self.gaze_pts_debug1.append(gaze1_3d)
            if nearest_intersection_point is not None:
                self.intersection_points_debug.append(nearest_intersection_point)

            self.sphere0["center"] = s0_center  # eye camera coordinates
            self.sphere0["radius"] = p0["sphere"]["radius"]
            self.sphere1["center"] = s1_center  # eye camera coordinates
            self.sphere1["radius"] = p1["sphere"]["radius"]

        if nearest_intersection_point is None:
            return None

        confidence = min(p0["confidence"], p1["confidence"])
        ts = (p0["timestamp"] + p1["timestamp"]) / 2.0
        g = {
            "topic": "gaze.3d.01.",
            "eye_centers_3d": {0: s0_center.tolist(), 1: s1_center.tolist()},
            "gaze_normals_3d": {0: s0_normal.tolist(), 1: s1_normal.tolist()},
            "gaze_point_3d": nearest_intersection_point.tolist(),
            "confidence": confidence,
            "timestamp": ts,
            "base_data": [p0, p1],
        }

        if self.backproject:
            g["norm_pos"] = image_point

        return g
Example #38
0
    def _map_binocular(self, p0, p1):

        if '3d' not in p0['method'] or '3d' not in p1['method']:
            return None

        #find the nearest intersection point of the two gaze lines
        #eye ball centers in world coords
        s0_center = self.eye0_to_World(np.array(p0['sphere']['center']))
        s1_center = self.eye1_to_World(np.array(p1['sphere']['center']))
        #eye line of sight in world coords
        s0_normal = np.dot(self.rotation_matricies[0],
                           np.array(p0['circle_3d']['normal']))
        s1_normal = np.dot(self.rotation_matricies[1],
                           np.array(p1['circle_3d']['normal']))

        # See Lech Swirski: "Gaze estimation on glasses-based stereoscopic displays"
        # Chapter: 7.4.2 Cyclopean gaze estimate

        #the cyclop is the avg of both lines of sight
        cyclop_normal = (s0_normal + s1_normal) / 2.
        cyclop_center = (s0_center + s1_center) / 2.

        # We use it to define a viewing plane.
        gaze_plane = np.cross(cyclop_normal, s1_center - s0_center)
        gaze_plane = gaze_plane / np.linalg.norm(gaze_plane)

        #project lines of sight onto the gaze plane
        s0_norm_on_plane = s0_normal - np.dot(gaze_plane,
                                              s0_normal) * gaze_plane
        s1_norm_on_plane = s1_normal - np.dot(gaze_plane,
                                              s1_normal) * gaze_plane

        #create gaze lines on this plane
        gaze_line0 = [s0_center, s0_center + s0_norm_on_plane]
        gaze_line1 = [s1_center, s1_center + s1_norm_on_plane]

        #find the intersection of left and right line of sight.
        nearest_intersection_point, intersection_distance = math_helper.nearest_intersection(
            gaze_line0, gaze_line1)
        if nearest_intersection_point is not None:
            cyclop_gaze = nearest_intersection_point - cyclop_center
            self.last_gaze_distance = np.sqrt(cyclop_gaze.dot(cyclop_gaze))
            image_point, _ = cv2.projectPoints(
                np.array([nearest_intersection_point]),
                np.array([0.0, 0.0, 0.0]), np.array([0.0, 0.0, 0.0]),
                self.camera_matrix, self.dist_coefs)
            image_point = image_point.reshape(-1, 2)
            image_point = normalize(image_point[0],
                                    self.world_frame_size,
                                    flip_y=True)
            image_point = _clamp_norm_point(image_point)

        if self.visualizer.window:
            gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
            gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
            self.gaze_pts_debug0.append(gaze0_3d)
            self.gaze_pts_debug1.append(gaze1_3d)
            if nearest_intersection_point is not None:
                self.intersection_points_debug.append(
                    nearest_intersection_point)

            self.sphere0['center'] = s0_center  #eye camera coordinates
            self.sphere0['radius'] = p0['sphere']['radius']
            self.sphere1['center'] = s1_center  #eye camera coordinates
            self.sphere1['radius'] = p1['sphere']['radius']

        if nearest_intersection_point is None:
            return None

        confidence = min(p0['confidence'], p1['confidence'])
        ts = (p0['timestamp'] + p1['timestamp']) / 2.
        g = {
            'topic': 'gaze',
            'norm_pos': image_point,
            'eye_centers_3d': {
                0: s0_center.tolist(),
                1: s1_center.tolist()
            },
            'gaze_normals_3d': {
                0: s0_normal.tolist(),
                1: s1_normal.tolist()
            },
            'gaze_point_3d': nearest_intersection_point.tolist(),
            'confidence': confidence,
            'timestamp': ts,
            'base_data': [p0, p1]
        }
        return g
Example #39
0
    def update(self, frame, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        if self.active:
            recent_pupil_positions = events['pupil_positions']

            gray_img = frame.gray

            if self.world_size is None:
                self.world_size = frame.width, frame.height

            self.markers = find_concetric_circles(gray_img, min_ring_count=3)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][
                    0]  #first marker, innermost ellipse, center
                self.pos = normalize(marker_pos, (frame.width, frame.height),
                                     flip_y=True)

            else:
                self.detected = False
                self.pos = None  #indicate that no reference is detected

            #tracking logic
            if self.detected:
                self.auto_stop += 1
                self.stop_marker_found = True

                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if not self.counter:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample {} datapoints"
                            .format(self.counter_max))
                        self.counter = self.counter_max

                if self.counter:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning(
                            "Marker moved to quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again."
                            .format((self.counter_max - self.counter)))
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = denormalize(
                            self.pos, (frame.width, frame.height), flip_y=True)
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if self.counter == 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
                                .format(self.counter_max))

            #always save pupil positions
            for pt in events.get('gaze_positions', []):
                if pt['confidence'] > self.pupil_confidence_threshold:
                    #we add an id for the calibration preprocess data to work as is usually expects pupil data.
                    pt['id'] = 0
                    self.gaze_list.append(pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'

            # stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()
        else:
            pass
Example #40
0
def world(
    timebase,
    eye_procs_alive,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    preferred_remote_port,
    hide_ui,
):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            "subject": "eye_process.should_start.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": delay,
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            "subject": "eye_process.should_stop.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": 0.2,
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {"subject": "set_detection_mapping_mode", "mode": new_mode}
        ipc_pub.notify(n)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # display
        import glfw
        from version_utils import VersionFormat
        from pyglui import ui, cygl, __version__ as pyglui_version

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.24"
        ), "pyglui out of date, please upgrade to newest version"
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic

        logger.info("Application Version: {}".format(version))
        logger.info("System Info: {}".format(get_system_info()))

        import audio

        # trigger pupil detector cpp build:
        import pupil_detectors

        del pupil_detectors

        # Plug-ins
        from plugin import (
            Plugin,
            System_Plugin_Base,
            Plugin_List,
            import_runtime_plugins,
        )
        from plugin_manager import Plugin_Manager
        from calibration_routines import (
            calibration_plugins,
            gaze_mapping_plugins,
            Calibration_Plugin,
            Gaze_Mapping_Plugin,
        )
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from pupil_remote import Pupil_Remote
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker_Online
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from frame_publisher import Frame_Publisher
        from blink_detection import Blink_Detection
        from video_capture import (
            source_classes,
            manager_classes,
            Base_Manager,
            Base_Source,
        )
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from audio_capture import Audio_Capture
        from accuracy_visualizer import Accuracy_Visualizer

        # from saccade_detector import Saccade_Detector
        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay
        from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (8, 90)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process they are accessible to all plugins
        g_pool = SimpleNamespace()
        g_pool.app = "capture"
        g_pool.process = "world"
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eye_procs_alive = eye_procs_alive
        g_pool.preferred_remote_port = preferred_remote_port

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, "plugins")
        )
        user_plugins = [
            Audio_Capture,
            Pupil_Groups,
            Frame_Publisher,
            Pupil_Remote,
            Time_Sync,
            Surface_Tracker_Online,
            Annotation_Capture,
            Log_History,
            Fixation_Detector,
            Blink_Detection,
            Remote_Recorder,
            Accuracy_Visualizer,
            Camera_Intrinsics_Estimation,
            Hololens_Relay,
            Online_Head_Pose_Tracker,
        ]

        system_plugins = (
            [
                Log_Display,
                Display_Recent_Gaze,
                Recorder,
                Pupil_Data_Relay,
                Plugin_Manager,
                System_Graphs,
            ]
            + manager_classes
            + source_classes
        )
        plugins = (
            system_plugins
            + user_plugins
            + runtime_plugins
            + calibration_plugins
            + gaze_mapping_plugins
        )
        user_plugins += [
            p
            for p in runtime_plugins
            if not isinstance(
                p,
                (
                    Base_Manager,
                    Base_Source,
                    System_Plugin_Base,
                    Calibration_Plugin,
                    Gaze_Mapping_Plugin,
                ),
            )
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_settings = {
            "preferred_names": [
                "Pupil Cam1 ID2",
                "Logitech Camera",
                "(046d:081d)",
                "C510",
                "B525",
                "C525",
                "C615",
                "C920",
                "C930e",
            ],
            "frame_size": (1280, 720),
            "frame_rate": 30,
        }

        default_plugins = [
            ("UVC_Source", default_capture_settings),
            ("Pupil_Data_Relay", {}),
            ("UVC_Manager", {}),
            ("Log_Display", {}),
            ("Dummy_Gaze_Mapper", {}),
            ("Display_Recent_Gaze", {}),
            ("Screen_Marker_Calibration", {}),
            ("Recorder", {}),
            ("Pupil_Remote", {}),
            ("Accuracy_Visualizer", {}),
            ("Plugin_Manager", {}),
            ("System_Graphs", {}),
        ]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()

            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_world")
        )
        if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", 0.8
        )
        g_pool.detection_mapping_mode = session_settings.get(
            "detection_mapping_mode", "3d"
        )
        g_pool.active_calibration_plugin = None
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode)

        def handle_notifications(noti):
            subject = noti["subject"]
            if subject == "set_detection_mapping_mode":
                if noti["mode"] == "2d":
                    if (
                        "Vector_Gaze_Mapper"
                        in g_pool.active_gaze_mapping_plugin.class_name
                    ):
                        logger.warning(
                            "The gaze mapper is not supported in 2d mode. Please recalibrate."
                        )
                        g_pool.plugins.add(g_pool.plugin_by_name["Dummy_Gaze_Mapper"])
                g_pool.detection_mapping_mode = noti["mode"]
            elif subject == "start_plugin":
                try:
                    g_pool.plugins.add(
                        g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})
                    )
                except KeyError as err:
                    logger.error(f"Attempt to load unknown plugin: {err}")
            elif subject == "stop_plugin":
                for p in g_pool.plugins:
                    if p.class_name == noti["name"]:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == "eye_process.started":
                noti = {
                    "subject": "set_detection_mapping_mode",
                    "mode": g_pool.detection_mapping_mode,
                }
                ipc_pub.notify(noti)
            elif subject == "set_min_calibration_confidence":
                g_pool.min_calibration_confidence = noti["value"]
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify(
                    {"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__}
                )
                for p in g_pool.plugins:
                    if (
                        p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify
                    ):
                        ipc_pub.notify(
                            {
                                "subject": "meta.doc",
                                "actor": p.class_name,
                                "doc": p.on_notify.__doc__,
                            }
                        )
            elif subject == "world_process.adapt_window_size":
                set_window_size()

        width, height = session_settings.get(
            "window_size", (1280 + icon_bar_width, 720)
        )

        # window and gl setup
        glfw.glfwInit()
        if hide_ui:
            glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0)  # hide window
        main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World")
        window_pos = session_settings.get("window_position", window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                camera_render_size[0]
                + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0})

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is opened, the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.6, 0.8, 1.0, 1.2, 1.4],
                label="Interface size",
            )
        )

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)
            on_resize(main_window, f_width, f_height)

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(
            ui.Selector("audio_mode", audio, selection=audio.audio_modes)
        )
        general_settings.append(
            ui.Selector(
                "detection_mapping_mode",
                g_pool,
                label="detection & mapping mode",
                setter=set_detection_mapping_mode,
                selection=["disabled", "2d", "3d"],
            )
        )
        general_settings.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: start_stop_eye(0, alive),
                getter=lambda: eye_procs_alive[0].value,
            )
        )
        general_settings.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: start_stop_eye(1, alive),
                getter=lambda: eye_procs_alive[1].value,
            )
        )

        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(g_pool.version))
        )
        general_settings.append(
            ui.Button("Restart with default settings", reset_restart)
        )

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins)
        )

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(True)

        # now that we have a proper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        if session_settings.get("eye1_process_alive", True):
            launch_eye_process(1, delay=0.6)
        if session_settings.get("eye0_process_alive", True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({"subject": "world_process.started"})
        logger.warning("Process started.")

        # Event loop
        while (
            not glfw.glfwWindowShouldClose(main_window) and not process_was_interrupted
        ):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # "blacklisted" events that were already sent
            del events["pupil"]
            del events["gaze"]
            # delete if exists. More expensive than del, so only use it when key might not exist
            events.pop("annotation", None)

            # send new events to ipc:
            if "frame" in events:
                del events["frame"]  # send explicitly with frame publisher
            if "depth_frame" in events:
                del events["depth_frame"]
            if "audio_packets" in events:
                del events["audio_packets"]
            del events["dt"]  # no need to send this
            for data in events.values():
                assert isinstance(data, (list, tuple))
                for d in data:
                    ipc_pub.send(d)

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            glfw.glfwPollEvents()
            if window_should_update() and gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                try:
                    clipboard = glfw.glfwGetClipboardString(main_window).decode()
                except AttributeError:  # clipboard is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(
                        main_window, user_input.clipboard.encode()
                    )

                for button, action, mods in user_input.buttons:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                glfw.glfwSwapBuffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)
        session_settings["eye0_process_alive"] = eye_procs_alive[0].value
        session_settings["eye1_process_alive"] = eye_procs_alive[1].value
        session_settings[
            "min_calibration_confidence"
        ] = g_pool.min_calibration_confidence
        session_settings["detection_mapping_mode"] = g_pool.detection_mapping_mode
        session_settings["audio_mode"] = audio.audio_mode

        if not hide_ui:
            glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
            session_window_size = glfw.glfwGetWindowSize(main_window)
            if 0 not in session_window_size:
                session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Capture crashed with trace:\n{}".format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "world_process.stopped"})
        sleep(1.0)
Example #41
0
def _create_ref_dict(ref, frame_size):
    return {
        "screen_pos": ref.screen_pos,
        "norm_pos": normalize(ref.screen_pos, frame_size, flip_y=True),
        "timestamp": ref.timestamp,
    }
 def on_pos(self, pos):
     self._last_mouse_pos = normalize(pos,
                                      self.g_pool.capture.frame_size,
                                      flip_y=True)
Example #43
0
    def recent_events(self, events):
        frame = events.get('frame')
        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']
            gray_img = frame.gray

            if self.clicks_to_close <= 0:
                self.stop()
                return

            # detect the marker
            self.markers = find_concetric_circles(gray_img, min_ring_count=4)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][
                    0]  # first marker, innermost ellipse,center
                self.pos = normalize(marker_pos, (frame.width, frame.height),
                                     flip_y=True)

            else:
                self.detected = False
                self.pos = None  # indicate that no reference is detected

            # only save a valid ref position if within sample window of calibration routine
            on_position = self.lead_in < self.screen_marker_state < (
                self.lead_in + self.sample_duration)

            if on_position and self.detected:
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            # always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if on_position and self.detected and events.get('fixations', []):
                fixation_boost = 5
                self.screen_marker_state = min(
                    self.sample_duration + self.lead_in,
                    self.screen_marker_state + fixation_boost)

            # Animate the screen marker
            if self.screen_marker_state < self.sample_duration + self.lead_in + self.lead_out:
                if self.detected or not on_position:
                    self.screen_marker_state += 1
            else:
                self.screen_marker_state = 0
                if not self.sites:
                    self.stop()
                    return
                self.active_site = self.sites.pop(0)
                logger.debug("Moving screen marker to site at {} {}".format(
                    *self.active_site))

            # use np.arrays for per element wise math
            self.display_pos = np.array(self.active_site)
            self.on_position = on_position
            self.button.status_text = '{} / {}'.format(self.active_site, 9)

        if self._window:
            self.gl_display_in_window()
Example #44
0
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
    hide_ui=False,
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
        ``set_detection_mapping_mode``: Sets detection method
        ``eye_process.should_stop``: Stops the eye process
        ``recording.started``: Starts recording eye video
        ``recording.stopped``: Stops recording eye video
        ``frame_publishing.started``: Starts frame publishing
        ``frame_publishing.stopped``: Stops frame publishing
        ``start_eye_plugin``: Start plugins in eye process

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport
        from ui_roi import UIRoi

        # monitoring
        import psutil

        # Plug-ins
        from plugin import Plugin_List

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, MPEG_Writer
        from ndsi import H264Writer
        from video_capture import source_classes, manager_classes

        from background_helper import IPC_Logging_Task_Proxy
        from pupil_detector_plugins import available_detector_plugins
        from pupil_detector_plugins.manager import PupilDetectorManager

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            # NOTE: Interrupt is handled in world/service/player which are responsible for
            # shutting down the eye process properly

        signal.signal(signal.SIGINT, interrupt_handler)

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = "capture"
        g_pool.eye_id = eye_id
        g_pool.process = f"eye{eye_id}"
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        default_detector_cls, available_detectors = available_detector_plugins(
        )
        plugins = (manager_classes + source_classes + available_detectors +
                   [PupilDetectorManager])
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        preferred_names = [
            f"Pupil Cam3 ID{eye_id}",
            f"Pupil Cam2 ID{eye_id}",
            f"Pupil Cam1 ID{eye_id}",
        ]
        if eye_id == 0:
            preferred_names += ["HD-6000"]
        default_capture_settings = (
            "UVC_Source",
            {
                "preferred_names": preferred_names,
                "frame_size": (320, 240),
                "frame_rate": 120,
            },
        )

        default_plugins = [
            # TODO: extend with plugins
            default_capture_settings,
            ("UVC_Manager", {}),
            # Detector needs to be loaded first to set `g_pool.pupil_detector`
            (default_detector_cls.__name__, {}),
            ("PupilDetectorManager", {}),
        ]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor

            active_window = glfw.glfwGetCurrentContext()
            glfw.glfwMakeContextCurrent(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x *= hdpi_factor
            y *= hdpi_factor
            g_pool.gui.update_mouse(x, y)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, g_pool.capture.frame_size)
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir,
                         "user_settings_eye{}".format(eye_id)))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.iconified = False
        g_pool.capture = None
        g_pool.flip = session_settings.get("flip", False)
        g_pool.display_mode = session_settings.get("display_mode",
                                                   "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image":
            "Raw eye camera image. This uses the least amount of CPU power",
            "roi":
            "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm":
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.glfwInit()
        if hide_ui:
            glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0)  # hide window
        title = "Pupil Capture - eye {}".format(eye_id)

        width, height = session_settings.get("window_size",
                                             (640 + icon_bar_width, 480))

        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get("window_position",
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(
            np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2],
                label="Interface Size",
            ))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width *= 2
            f_height *= 2
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        def uroi_on_mouse_button(button, action, mods):
            if g_pool.display_mode == "roi":
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    x *= hdpi_factor
                    y *= hdpi_factor
                    pos = normalize((x, y), camera_render_size)
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(
                        pos,
                        g_pool.capture.frame_size)  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,
                                                     g_pool.u_r.handle_size,
                                                     g_pool.u_r.handle_size):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            ))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])

        general_settings.append(g_pool.display_mode_info)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)
        toggle_general_settings(False)

        plugins_to_load = session_settings.get("loaded_plugins",
                                               default_plugins)
        if overwrite_cap_settings:
            # Ensure that overwrite_cap_settings takes preference over source plugins
            # with incorrect settings that were loaded from session settings.
            plugins_to_load.append(overwrite_cap_settings)

        g_pool.plugins = Plugin_List(g_pool, plugins_to_load)

        g_pool.writer = None

        g_pool.u_r = UIRoi(
            (g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get("roi")
        if roi_user_settings and tuple(
                roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        record_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        start_time_synced = notification["start_time_synced"]
                        logger.info(
                            "Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(record_path,
                                                  "eye{}.mp4".format(eye_id))
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(
                                video_path, start_time_synced)
                        elif hasattr(g_pool.capture._recent_frame,
                                     "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = MPEG_Writer(
                                video_path, start_time_synced)
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify({
                        "subject": "meta.doc",
                        "actor": "eye{}".format(eye_id),
                        "doc": eye.__doc__,
                    })
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (subject.startswith("start_eye_plugin")
                      and notification["target"] == g_pool.process):
                    try:
                        g_pool.plugins.add(
                            g_pool.plugin_by_name[notification["name"]],
                            notification.get("args", {}),
                        )
                    except KeyError as err:
                        logger.error(f"Attempt to load unknown plugin: {err}")

                for plugin in g_pool.plugins:
                    plugin.on_notify(notification)

            event = {}
            for plugin in g_pool.plugins:
                plugin.recent_events(event)

            frame = event.get("frame")
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                # TODO: Roi should be its own plugin. This way we could put it at the
                # appropriate order for recent_events() to process frame resolution
                # changes immediately after the backend.
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
                        f_height,
                        f_width,
                ):
                    g_pool.pupil_detector.on_resolution_change(
                        (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
                        g_pool.capture.frame_size,
                    )
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.
                                format(type(frame), frame_publish_format))
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send({
                            "topic":
                            "frame.eye.{}".format(eye_id),
                            "width":
                            frame.width,
                            "height":
                            frame.height,
                            "index":
                            frame.index,
                            "timestamp":
                            frame.timestamp,
                            "format":
                            frame_publish_format,
                            "__raw_data__": [data],
                        })

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                result = event.get("pupil_detection_result", None)
                if result is not None:
                    pupil_socket.send(result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    glViewport(0, 0, *camera_render_size)
                    for p in g_pool.plugins:
                        p.gl_display()

                    glViewport(0, 0, *camera_render_size)
                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == "roi":
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    glViewport(0, 0, *window_size)
                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    try:
                        clipboard = glfw.glfwGetClipboardString(
                            main_window).decode()
                    except AttributeError:  # clipboard is None, might happen on startup
                        clipboard = ""
                    g_pool.gui.update_clipboard(clipboard)
                    user_input = g_pool.gui.update()
                    if user_input.clipboard != clipboard:
                        # only write to clipboard if content changed
                        glfw.glfwSetClipboardString(
                            main_window, user_input.clipboard.encode())

                    for button, action, mods in user_input.buttons:
                        x, y = glfw.glfwGetCursorPos(main_window)
                        pos = x * hdpi_factor, y * hdpi_factor
                        pos = normalize(pos, camera_render_size)
                        # Position in img pixels
                        pos = denormalize(pos, g_pool.capture.frame_size)

                        # TODO: remove when ROI is plugin
                        uroi_on_mouse_button(button, action, mods)

                        for plugin in g_pool.plugins:
                            if plugin.on_click(pos, button, action):
                                break

                    for key, scancode, action, mods in user_input.keys:
                        for plugin in g_pool.plugins:
                            if plugin.on_key(key, scancode, action, mods):
                                break

                    for char_ in user_input.chars:
                        for plugin in g_pool.plugins:
                            if plugin.on_char(char_):
                                break

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer.release()
            g_pool.writer = None

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        # save session persistent settings
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["roi"] = g_pool.u_r.get()
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)

        if not hide_ui:
            glfw.glfwRestoreWindow(
                main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.glfwGetWindowPos(
                main_window)
            session_window_size = glfw.glfwGetWindowSize(main_window)
            if 0 not in session_window_size:
                session_settings["window_size"] = session_window_size

        session_settings.close()

        for plugin in g_pool.plugins:
            plugin.alive = False
        g_pool.plugins.clean()

        glfw.glfwDestroyWindow(main_window)
        g_pool.gui.terminate()
        glfw.glfwTerminate()
        logger.info("Process shutting down.")
Example #45
0
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
          user_dir, version):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    # logger.setLevel(logging.DEBUG)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            'subject': 'eye_process.should_start.{}'.format(eye_id),
            'eye_id': eye_id,
            'delay': delay
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            'subject': 'eye_process.should_stop.{}'.format(eye_id),
            'eye_id': eye_id,
            'delay': 0.2
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
        ipc_pub.notify(n)

    try:

        # display
        import glfw
        from version_utils import VersionFormat
        from pyglui import ui, cygl, __version__ as pyglui_version
        assert VersionFormat(pyglui_version) >= VersionFormat(
            '1.9'), 'pyglui out of date, please upgrade to newest version'
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic
        logger.info('Application Version: {}'.format(version))
        logger.info('System Info: {}'.format(get_system_info()))

        import audio

        # trigger pupil detector cpp build:
        import pupil_detectors
        del pupil_detectors

        # Plug-ins
        from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from pupil_remote import Pupil_Remote
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from frame_publisher import Frame_Publisher
        from blink_detection import Blink_Detection
        from video_capture import source_classes, manager_classes, Base_Manager, Base_Source
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from audio_capture import Audio_Capture
        from accuracy_visualizer import Accuracy_Visualizer
        from emse_numero_plugin import Emse_numero_plugin
        # from saccade_detector import Saccade_Detector
        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation

        # UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == 'Windows':
            scroll_factor = 10.0
            window_position_default = (8, 31)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process they are accesible to all plugins
        g_pool = Global_Container()
        g_pool.app = 'capture'
        g_pool.process = 'world'
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eyes_are_alive = eyes_are_alive

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, 'plugins'))
        user_plugins = [
            Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote,
            Time_Sync, Surface_Tracker, Annotation_Capture, Log_History,
            Fixation_Detector, Blink_Detection, Remote_Recorder,
            Accuracy_Visualizer, Camera_Intrinsics_Estimation,
            Emse_numero_plugin
        ]
        system_plugins = [
            Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay,
            Plugin_Manager, System_Graphs
        ] + manager_classes + source_classes
        plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins
        user_plugins += [
            p for p in runtime_plugins
            if not isinstance(p, (Base_Manager, Base_Source,
                                  System_Plugin_Base, Calibration_Plugin,
                                  Gaze_Mapping_Plugin))
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_settings = {
            'preferred_names': [
                "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510",
                "B525", "C525", "C615", "C920", "C930e"
            ],
            'frame_size': (1280, 720),
            'frame_rate':
            30
        }

        default_plugins = [("UVC_Source", default_capture_settings),
                           ('Pupil_Data_Relay', {}), ('UVC_Manager', {}),
                           ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
                           ('Display_Recent_Gaze', {}),
                           ('Screen_Marker_Calibration', {}), ('Recorder', {}),
                           ('Pupil_Remote', {}), ('Plugin_Manager', {}),
                           ('System_Graphs', {})]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode('utf-8') for x in range(count)]
            for p in g_pool.plugins:
                p.on_drop(paths)

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_world'))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.detection_mapping_mode = session_settings.get(
            'detection_mapping_mode', '3d')
        g_pool.active_calibration_plugin = None
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.audio_mode = session_settings.get('audio_mode',
                                                audio.default_audio_mode)

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'set_detection_mapping_mode':
                if n['mode'] == '2d':
                    if ("Vector_Gaze_Mapper"
                            in g_pool.active_gaze_mapping_plugin.class_name):
                        logger.warning(
                            "The gaze mapper is not supported in 2d mode. Please recalibrate."
                        )
                        g_pool.plugins.add(
                            g_pool.plugin_by_name['Dummy_Gaze_Mapper'])
                g_pool.detection_mapping_mode = n['mode']
            elif subject == 'start_plugin':
                g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
                                   args=n.get('args', {}))
            elif subject == 'stop_plugin':
                for p in g_pool.plugins:
                    if p.class_name == n['name']:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == 'eye_process.started':
                n = {
                    'subject': 'set_detection_mapping_mode',
                    'mode': g_pool.detection_mapping_mode
                }
                ipc_pub.notify(n)
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({
                    'subject': 'meta.doc',
                    'actor': g_pool.app,
                    'doc': world.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            'subject': 'meta.doc',
                            'actor': p.class_name,
                            'doc': p.on_notify.__doc__
                        })

        # window and gl setup
        glfw.glfwInit()
        width, height = session_settings.get('window_size',
                                             (1280 + icon_bar_width, 720))
        main_window = glfw.glfwCreateWindow(width, height,
                                            "Pupil Capture - World")
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = camera_render_size[0] + \
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \
                glfw.glfwGetFramebufferSize(main_window)[1]
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({
                'subject': 'world_process.should_start',
                'delay': 2.
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-400, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos='hidden')
        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
                                             (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.6, .8, 1., 1.2, 1.4],
                        label='Interface size'))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings.append(ui.Button('Reset window size',
                                          set_window_size))
        general_settings.append(
            ui.Selector('audio_mode', audio, selection=audio.audio_modes))
        general_settings.append(
            ui.Selector('detection_mapping_mode',
                        g_pool,
                        label='detection & mapping mode',
                        setter=set_detection_mapping_mode,
                        selection=['2d', '3d']))
        general_settings.append(
            ui.Switch('eye0_process',
                      label='Detect eye 0',
                      setter=lambda alive: start_stop_eye(0, alive),
                      getter=lambda: eyes_are_alive[0].value))
        general_settings.append(
            ui.Switch('eye1_process',
                      label='Detect eye 1',
                      setter=lambda alive: start_stop_eye(1, alive),
                      getter=lambda: eyes_are_alive[1].value))

        general_settings.append(
            ui.Info_Text('Capture Version: {}'.format(g_pool.version)))
        general_settings.append(
            ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed',
                       general_settings,
                       label=chr(0xe8b8),
                       on_val=False,
                       off_val=True,
                       setter=toggle_general_settings,
                       label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(False)

        # now the we have  aproper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        if session_settings.get('eye1_process_alive', False):
            launch_eye_process(1, delay=0.6)
        if session_settings.get('eye0_process_alive', True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({'subject': 'world_process.started'})
        logger.warning('Process started.')

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            #a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # send new events to ipc:
            del events['pupil_positions']  # already on the wire
            del events['gaze_positions']  # sent earlier
            if 'frame' in events:
                del events['frame']  # send explicity with frame publisher
            if 'depth_frame' in events:
                del events['depth_frame']
            if 'audio_packets' in events:
                del events['audio_packets']
            del events['dt']  # no need to send this
            for topic, data in events.items():
                assert (isinstance(data, (list, tuple)))
                for d in data:
                    ipc_pub.send(topic, d)

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if window_should_update() and gl_utils.is_window_visible(
                    main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                unused_elements = g_pool.gui.update()
                for button, action, mods in unused_elements.buttons:
                    pos = glfw.glfwGetCursorPos(main_window)
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in unused_elements.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in unused_elements.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                glfw.glfwSwapBuffers(main_window)
            glfw.glfwPollEvents()

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings['eye0_process_alive'] = eyes_are_alive[0].value
        session_settings['eye1_process_alive'] = eyes_are_alive[1].value
        session_settings[
            'detection_mapping_mode'] = g_pool.detection_mapping_mode
        session_settings['audio_mode'] = audio.audio_mode
        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Capture crashed with trace:\n{}'.format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'world_process.stopped'})
        sleep(1.0)
Example #46
0
def player(rec_dir, ipc_pub_url, ipc_sub_url,
           ipc_push_url, user_dir, app_version):
    # general imports
    import logging
    import errno
    from glob import glob
    from copy import deepcopy
    from time import time
    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)



    # imports
    from file_methods import Persistent_Dict, load_object
    import numpy as np

    # display
    import glfw
    # check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version

    from pyglui import ui, graph, cygl
    from pyglui.cygl.utils import Named_Texture
    import gl_utils
    # capture
    from video_capture import File_Source, EndofVideoFileError, FileSeekError

    # helpers/utils
    from version_utils import VersionFormat
    from methods import normalize, denormalize, delta_t, get_system_info
    from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

    # monitoring
    import psutil

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
    from vis_circle import Vis_Circle
    from vis_cross import Vis_Cross
    from vis_polyline import Vis_Polyline
    from vis_light_points import Vis_Light_Points
    from vis_watermark import Vis_Watermark
    from vis_fixation import Vis_Fixation
    from vis_scan_path import Vis_Scan_Path
    from vis_eye_video_overlay import Vis_Eye_Video_Overlay
    from seek_bar import Seek_Bar
    from trim_marks import Trim_Marks
    from video_export_launcher import Video_Export_Launcher
    from offline_surface_tracker import Offline_Surface_Tracker
    from marker_auto_trim_marks import Marker_Auto_Trim_Marks
    from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
    # from manual_gaze_correction import Manual_Gaze_Correction
    from batch_exporter import Batch_Exporter
    from log_display import Log_Display
    from annotations import Annotation_Player
    from raw_data_exporter import Raw_Data_Exporter
    from log_history import Log_History
    from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
    from gaze_producers import Gaze_From_Recording, Offline_Calibration

    assert pyglui_version >= '1.6'

    runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
                               Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector,
                               Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher,
                               Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player,
                               Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection,
                               Gaze_From_Recording, Offline_Calibration] + runtime_plugins

    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_window_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_window_char(window, char):
        g_pool.gui.update_char(char)

    def on_window_mouse_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
                glfw.glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
                  if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    timestamps = np.load(timestamps_path)

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url

    # Initialize capture
    cap = File_Source(g_pool, video_path, timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if VersionFormat(session_settings.get("version", '0.0')) != app_version:
        logger.info("Session setting are a different version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    glfw.glfwInit()
    main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    # load pupil_positions, gaze_positions
    g_pool.pupil_data = load_object(pupil_data_path)
    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.get_timestamp = lambda: 0.
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)

    g_pool.pupil_positions = []
    g_pool.gaze_positions = []
    g_pool.fixations = []

    g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
    g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
    g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() + 1)
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() - 1)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.ipc_pub.notify(notification)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(*export_range))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.ipc_pub.notify(notification)

    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
    g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    g_pool.main_menu.append(ui.Info_Text('Open plugins'))

    selector_label = "Select to load"

    def append_selector(label, plugins):
        plugins.sort(key=lambda p: p.__name__)
        plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
        g_pool.main_menu.append(ui.Selector(label,
                                            selection=[selector_label] + plugins,
                                            labels=[selector_label] + plugin_labels,
                                            setter=open_plugin,
                                            getter=lambda: selector_label))

    base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base]
    base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
    launchable = user_launchable_plugins.copy()
    for base_class, label in zip(base_plugins, base_labels):
        member_plugins = []
        for p in user_launchable_plugins:
            if issubclass(p, base_class):
                member_plugins.append(p)
                launchable.remove(p)
        append_selector(label, member_plugins)

    # launchable only contains plugins that could not be assigned to any of the above categories
    append_selector('Other', launchable)

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=glfw.GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=glfw.GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=glfw.GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}),
                       ('Vis_Circle', {}), ('Video_Export_Launcher', {}),
                       ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetKeyCallback(main_window, on_window_key)
    glfw.glfwSetCharCallback(main_window, on_window_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)
    glfw.glfwSetDropCallback(main_window, on_drop)

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"
    g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]

    # trigger on_resize
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'start_plugin':
            g_pool.plugins.add(
                plugin_by_name[n['name']], args=n.get('args', {}))
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({'subject': 'meta.doc',
                            'actor': g_pool.app,
                            'doc': player.__doc__})
            for p in g_pool.plugins:
                if (p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify):
                    ipc_pub.notify({'subject': 'meta.doc',
                                    'actor': p.class_name,
                                    'doc': p.on_notify.__doc__})

    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            for p in g_pool.plugins:
                p.on_notify(n)

        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        events['frame'] = frame
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
                ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        unused_elements = g_pool.gui.update()
        for b in unused_elements.buttons:
            button, action, mods = b
            pos = glfw.glfwGetCursorPos(main_window)
            pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
            pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
            for p in g_pool.plugins:
                p.on_click(pos, button, action)

        for key, scancode, action, mods in unused_elements.keys:
            for p in g_pool.plugins:
                p.on_key(key, scancode, action, mods)

        for char_ in unused_elements.chars:
            for p in g_pool.plugins:
                p.on_char(char_)

        # present frames at appropriate speed
        cap.wait(frame)

        glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'player_process.stopped'})
Example #47
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime
    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        # display
        import glfw
        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils
        # capture
        from video_capture import init_playback_source

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation
        # from vis_scan_path import Vis_Scan_Path
        from vis_eye_video_overlay import Vis_Eye_Video_Overlay
        from seek_control import Seek_Control
        from video_export_launcher import Video_Export_Launcher
        from offline_surface_tracker import Offline_Surface_Tracker
        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from batch_exporter import Batch_Exporter, Batch_Export
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producers import Gaze_From_Recording, Offline_Calibration
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from imotions_exporter import iMotions_Exporter
        from eye_video_exporter import Eye_Video_Exporter

        assert VersionFormat(pyglui_version) >= VersionFormat(
            '1.23'), 'pyglui out of date, please upgrade to newest version'

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, 'plugins'))
        system_plugins = [
            Log_Display, Seek_Control, Plugin_Manager, System_Graphs,
            Batch_Export, System_Timelines, Audio_Playback
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Vis_Eye_Video_Overlay,
            # Vis_Scan_Path,
            Offline_Fixation_Detector,
            Offline_Blink_Detection,
            Batch_Exporter,
            Video_Export_Launcher,
            Offline_Surface_Tracker,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            Gaze_From_Recording,
            iMotions_Exporter,
            Eye_Video_Exporter,
            Offline_Calibration
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            for x in range(count):
                new_rec_dir = paths[x].decode('utf-8')
                if pm.is_pupil_rec_dir(new_rec_dir):
                    logger.debug(
                        "Starting new session with '{}'".format(new_rec_dir))
                    ipc_pub.notify({
                        "subject": "player_drop_process.should_start",
                        "rec_dir": new_rec_dir
                    })
                    glfw.glfwSetWindowShouldClose(window, True)
                else:
                    logger.error("'{}' is not a valid pupil recording".format(
                        new_rec_dir))

        tick = delta_t()

        def get_dt():
            return next(tick)

        meta_info = pm.load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info('Application Version: {}'.format(app_version))
        logger.info('System Info: {}'.format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = Global_Container()
        g_pool.app = 'player'
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg', '.fake')
        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in valid_ext
        ][0]
        init_playback_source(g_pool,
                             timing='external',
                             source_path=video_path,
                             buffered_decoding=True)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get('window_size', (width, height))

        window_pos = session_settings.get('window_position',
                                          window_position_default)
        window_name = "Pupil Player: {} - {}".format(
            meta_info["Recording Name"],
            os.path.split(rec_dir)[-1])

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, window_name, None,
                                            None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1])
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.binocular = meta_info.get('Eye Mode',
                                         'monocular') == 'binocular'
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            'min_data_confidence', 0.6)
        g_pool.min_calibration_confidence = session_settings.get(
            'min_calibration_confidence', 0.8)

        # populated by producers
        g_pool.pupil_positions = pm.Bisector()
        g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {'subject': 'min_data_confidence_changed'}
            notification['_notify_time_'] = time() + .8
            g_pool.ipc_pub.notify(notification)

        def open_plugin(plugin):
            if plugin == "Select to load":
                return
            g_pool.plugins.add(plugin)

        def purge_plugins():
            for p in g_pool.plugins:
                if p.__class__ in user_plugins:
                    p.alive = False
            g_pool.plugins.clean()

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop

            export_dir = os.path.join(g_pool.rec_dir, 'exports')
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                'Player Software Version':
                str(g_pool.version),
                'Data Format Version':
                meta_info['Data Format Version'],
                'Export Date':
                strftime("%d.%m.%Y", localtime()),
                'Export Time':
                strftime("%H:%M:%S", localtime()),
                'Frame Index Range:':
                g_pool.seek_control.get_frame_index_trim_range_string(),
                'Relative Time Range':
                g_pool.seek_control.get_rel_time_trim_range_string(),
                'Absolute Time Range':
                g_pool.seek_control.get_abs_time_trim_range_string()
            }
            with open(os.path.join(export_dir, 'export_info.csv'), 'w') as csv:
                write_key_value_file(csv, export_info)

            notification = {
                'subject': 'should_export',
                'range': export_range,
                'export_dir': export_dir
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({
                'subject': 'player_process.should_start',
                'rec_dir': rec_dir,
                'delay': 2.
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos='hidden')
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu('User Timelines',
                                                 pos=(0., -150.),
                                                 size=(0., 0.),
                                                 header_pos='headline')
        g_pool.user_timelines.color = RGBA(a=0.)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(ui.Button('Reset window size',
                                          set_window_size))
        general_settings.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.8, .9, 1., 1.1, 1.2] +
                        list(np.arange(1.5, 5.1, .5)),
                        label='Interface Size'))
        general_settings.append(
            ui.Info_Text('Player Version: {}'.format(g_pool.version)))
        general_settings.append(
            ui.Info_Text('Capture Version: {}'.format(
                meta_info['Capture Software Version'])))
        general_settings.append(
            ui.Info_Text('Data Format Version: {}'.format(
                meta_info['Data Format Version'])))

        general_settings.append(
            ui.Info_Text(
                'High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence.'
            ))
        general_settings.append(
            ui.Slider('min_data_confidence',
                      g_pool,
                      setter=set_data_confidence,
                      step=.05,
                      min=0.0,
                      max=1.0,
                      label='Minimum data confidence'))

        general_settings.append(
            ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed',
                       general_settings,
                       label=chr(0xe8b8),
                       on_val=False,
                       off_val=True,
                       setter=toggle_general_settings,
                       label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb('export',
                                        label=chr(0xe2c5),
                                        getter=lambda: False,
                                        setter=do_export,
                                        hotkey='e',
                                        label_font='pupil_icons')
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}),
                           ('Log_Display', {}), ('Raw_Data_Exporter', {}),
                           ('Vis_Polyline', {}), ('Vis_Circle', {}),
                           ('System_Graphs', {}), ('System_Timelines', {}),
                           ('Video_Export_Launcher', {}),
                           ('Pupil_From_Recording', {}),
                           ('Gaze_From_Recording', {}), ('Audio_Playback', {})]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                'rel_time_trim_section',
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label='Relative time range to export'))
        general_settings.insert(
            -1,
            ui.Text_Input(
                'frame_idx_trim_section',
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label='Frame index range to export'))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'start_plugin':
                g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
                                   args=n.get('args', {}))
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({
                    'subject': 'meta.doc',
                    'actor': g_pool.app,
                    'doc': player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            'subject': 'meta.doc',
                            'actor': p.class_name,
                            'doc': p.on_notify.__doc__
                        })

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events['pupil'] = []
            events['gaze'] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ''
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in user_input.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in user_input.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                # present frames at appropriate speed
                g_pool.seek_control.wait(events['frame'].timestamp)
                glfw.glfwSwapBuffers(main_window)

            glfw.glfwPollEvents()

        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['min_data_confidence'] = g_pool.min_data_confidence
        session_settings[
            'min_calibration_confidence'] = g_pool.min_calibration_confidence
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = str(g_pool.version)

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings['window_size'] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Player crashed with trace:\n{}'.format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'player_process.stopped'})
        sleep(1.0)
Example #48
0
 def on_pos(window, x, y):
     norm_pos = normalize((x, y), glfwGetWindowSize(window))
     fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window))
     if atb.TwMouseMotion(int(fb_x), int(fb_y)):
         pass
Example #49
0
    def recent_events(self, events):
        frame = events.get('frame')
        if not frame:
            return
        self.img_shape = frame.img.shape
        self.update_marker_cache()
        # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
        self.markers = self.cache[frame.index]
        if self.markers is False:
            self.markers = []
            # tell precacher that it better have every thing from here on analyzed
            self.seek_marker_cacher(frame.index)

        events['surfaces'] = []
        # locate surfaces
        for s in self.surfaces:
            if not s.locate_from_cache(frame.index):
                s.locate(self.markers, self.min_marker_perimeter,
                         self.min_id_confidence)
            if s.detected:
                events['surfaces'].append({
                    'name':
                    s.name,
                    'uid':
                    s.uid,
                    'm_to_screen':
                    s.m_to_screen.tolist(),
                    'm_from_screen':
                    s.m_from_screen.tolist(),
                    'gaze_on_srf':
                    s.gaze_on_srf,
                    'timestamp':
                    frame.timestamp,
                    'camera_pose_3d':
                    s.camera_pose_3d.tolist()
                    if s.camera_pose_3d is not None else None
                })

        if self.mode == "Show marker IDs":
            draw_markers(frame.img, self.markers)

        elif self.mode == "Show Markers and Surfaces":
            # edit surfaces by user
            if self.edit_surf_verts:
                window = glfwGetCurrentContext()
                pos = glfwGetCursorPos(window)
                pos = normalize(pos,
                                self.g_pool.camera_render_size,
                                flip_y=True)
                for s, v_idx in self.edit_surf_verts:
                    if s.detected:
                        new_pos = s.img_to_ref_surface(np.array(pos))
                        s.move_vertex(v_idx, new_pos)
            else:
                # update srf with no or invald cache:
                for s in self.surfaces:
                    if s.cache == None:
                        s.init_cache(self.cache, self.min_marker_perimeter,
                                     self.min_id_confidence)
                        self.notify_all({
                            'subject': 'surfaces_changed',
                            'delay': 1
                        })

        # allow surfaces to open/close windows
        for s in self.surfaces:
            if s.window_should_close:
                s.close_window()
            if s.window_should_open:
                s.open_window()
    def update(self, frame, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        if self.active:
            recent_pupil_positions = events['pupil_positions']

            gray_img = frame.gray

            if self.world_size is None:
                self.world_size = frame.width, frame.height

            self.candidate_ellipses = get_candidate_ellipses(
                gray_img,
                area_threshold=self.area_threshold,
                dist_threshold=self.dist_threshold,
                min_ring_count=5,
                visual_debug=self.show_edges)

            if len(self.candidate_ellipses) > 0:
                self.detected = True
                marker_pos = self.candidate_ellipses[0][0]
                self.pos = normalize(marker_pos, (frame.width, frame.height),
                                     flip_y=True)

            else:
                self.detected = False
                self.pos = None  #indicate that no reference is detected

            # center dark or white?
            if self.detected:
                second_ellipse = self.candidate_ellipses[1]
                col_slice = int(second_ellipse[0][0] - second_ellipse[1][0] /
                                2), int(second_ellipse[0][0] +
                                        second_ellipse[1][0] / 2)
                row_slice = int(second_ellipse[0][1] - second_ellipse[1][1] /
                                2), int(second_ellipse[0][1] +
                                        second_ellipse[1][1] / 2)
                marker_gray = gray_img[slice(*row_slice), slice(*col_slice)]
                avg = cv2.mean(marker_gray)[0]  #CV2 fn return has changed!
                center = marker_gray[second_ellipse[1][1] / 2,
                                     second_ellipse[1][0] / 2]
                rel_shade = center - avg

                #auto_stop logic
                if rel_shade > 30:
                    #bright marker center found
                    self.auto_stop += 1
                    self.stop_marker_found = True

                else:
                    self.auto_stop = 0
                    self.stop_marker_found = False

            #tracking logic
            if self.detected and not self.stop_marker_found:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if not self.counter:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample %s datapoints"
                            % self.counter_max)
                        self.counter = self.counter_max

                if self.counter:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.debug(
                            "Marker moved to quickly: Aborted sample. Sampled %s datapoints. Looking for steady marker again."
                            % (self.counter_max - self.counter))
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if self.counter == 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled %s datapoints. Stopping to sample. Looking for steady marker again."
                                % self.counter_max)

            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.g_pool.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'

            #stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

        else:
            pass
Example #51
0
def world(
    timebase,
    eye_procs_alive,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    preferred_remote_port,
    hide_ui,
    debug,
):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``eye_process.started``
        ``start_plugin``
        ``should_stop``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            "subject": "eye_process.should_start.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": delay,
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            "subject": "eye_process.should_stop.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": 0.2,
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def detection_enabled_getter() -> int:
        return int(g_pool.pupil_detection_enabled)

    def detection_enabled_setter(value: int):
        is_on = bool(value)
        g_pool.pupil_detection_enabled = is_on
        n = {"subject": "pupil_detector.set_enabled", "value": is_on}
        ipc_pub.notify(n)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        from OpenGL.GL import GL_COLOR_BUFFER_BIT

        # display
        import glfw
        from gl_utils import GLFWErrorReporting

        GLFWErrorReporting.set_default()

        from version_utils import parse_version
        from pyglui import ui, cygl, __version__ as pyglui_version

        assert parse_version(pyglui_version) >= parse_version(
            "1.27"), "pyglui out of date, please upgrade to newest version"
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic

        logger.info("Application Version: {}".format(version))
        logger.info("System Info: {}".format(get_system_info()))
        logger.debug(f"Debug flag: {debug}")

        import audio

        # Plug-ins
        from plugin import (
            Plugin,
            System_Plugin_Base,
            Plugin_List,
            import_runtime_plugins,
        )
        from plugin_manager import Plugin_Manager
        from calibration_choreography import (
            available_calibration_choreography_plugins,
            CalibrationChoreographyPlugin,
            patch_loaded_plugins_with_choreography_plugin,
        )

        available_choreography_plugins = available_calibration_choreography_plugins(
        )

        from gaze_mapping import registered_gazer_classes
        from gaze_mapping.gazer_base import GazerBase
        from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from network_api import NetworkApiPlugin
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker_Online
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from blink_detection import Blink_Detection
        from video_capture import (
            source_classes,
            manager_classes,
            Base_Manager,
            Base_Source,
        )
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from accuracy_visualizer import Accuracy_Visualizer

        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay
        from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (8, 90)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        content_scale = 1.0

        # g_pool holds variables for this process they are accessible to all plugins
        g_pool = SimpleNamespace()
        g_pool.debug = debug
        g_pool.app = "capture"
        g_pool.process = "world"
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eye_procs_alive = eye_procs_alive
        g_pool.preferred_remote_port = preferred_remote_port

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, "plugins"))
        runtime_plugins = [
            p for p in runtime_plugins
            if not issubclass(p, PupilDetectorPlugin)
        ]
        user_plugins = [
            Pupil_Groups,
            NetworkApiPlugin,
            Time_Sync,
            Surface_Tracker_Online,
            Annotation_Capture,
            Log_History,
            Fixation_Detector,
            Blink_Detection,
            Remote_Recorder,
            Accuracy_Visualizer,
            Camera_Intrinsics_Estimation,
            Hololens_Relay,
            Online_Head_Pose_Tracker,
        ]

        system_plugins = ([
            Log_Display,
            Display_Recent_Gaze,
            Recorder,
            Pupil_Data_Relay,
            Plugin_Manager,
            System_Graphs,
        ] + manager_classes + source_classes)
        plugins = (system_plugins + user_plugins + runtime_plugins +
                   available_choreography_plugins + registered_gazer_classes())
        user_plugins += [
            p for p in runtime_plugins if not isinstance(
                p,
                (
                    Base_Manager,
                    Base_Source,
                    System_Plugin_Base,
                    CalibrationChoreographyPlugin,
                    GazerBase,
                ),
            )
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_name = "UVC_Source"
        default_capture_settings = {
            "preferred_names": [
                "Pupil Cam1 ID2",
                "Logitech Camera",
                "(046d:081d)",
                "C510",
                "B525",
                "C525",
                "C615",
                "C920",
                "C930e",
            ],
            "frame_size": (1280, 720),
            "frame_rate":
            30,
        }

        default_plugins = [
            (default_capture_name, default_capture_settings),
            ("Pupil_Data_Relay", {}),
            ("UVC_Manager", {}),
            ("NDSI_Manager", {}),
            ("HMD_Streaming_Manager", {}),
            ("File_Manager", {}),
            ("Log_Display", {}),
            ("Dummy_Gaze_Mapper", {}),
            ("Display_Recent_Gaze", {}),
            # Calibration choreography plugin is added below by calling
            # patch_loaded_plugins_with_choreography_plugin
            ("Recorder", {}),
            ("NetworkApiPlugin", {}),
            ("Fixation_Detector", {}),
            ("Blink_Detection", {}),
            ("Accuracy_Visualizer", {}),
            ("Plugin_Manager", {}),
            ("System_Graphs", {}),
        ]

        def consume_events_and_render_buffer():
            gl_utils.glViewport(0, 0, *camera_render_size)
            for p in g_pool.plugins:
                p.gl_display()

            gl_utils.glViewport(0, 0, *window_size)
            try:
                clipboard = glfw.get_clipboard_string(main_window).decode()
            except (AttributeError, glfw.GLFWError):
                # clipboard is None, might happen on startup
                clipboard = ""
            g_pool.gui.update_clipboard(clipboard)
            user_input = g_pool.gui.update()
            if user_input.clipboard != clipboard:
                # only write to clipboard if content changed
                glfw.set_clipboard_string(main_window, user_input.clipboard)

            for button, action, mods in user_input.buttons:
                x, y = glfw.get_cursor_pos(main_window)
                pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                    main_window, x, y, cached_scale=None)
                pos = normalize(pos, camera_render_size)
                # Position in img pixels
                pos = denormalize(pos, g_pool.capture.frame_size)

                for plugin in g_pool.plugins:
                    if plugin.on_click(pos, button, action):
                        break

            for key, scancode, action, mods in user_input.keys:
                for plugin in g_pool.plugins:
                    if plugin.on_key(key, scancode, action, mods):
                        break

            for char_ in user_input.chars:
                for plugin in g_pool.plugins:
                    if plugin.on_char(char_):
                        break

            glfw.swap_buffers(main_window)

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal content_scale
            if w == 0 or h == 0:
                return

            # Always clear buffers on resize to make sure that there are no overlapping
            # artifacts from previous frames.
            gl_utils.glClear(GL_COLOR_BUFFER_BIT)
            gl_utils.glClearColor(0, 0, 0, 1)

            content_scale = gl_utils.get_content_scale(window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(window)
            g_pool.gui.scale = content_scale
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()

            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

            # Minimum window size required, otherwise parts of the UI can cause openGL
            # issues with permanent effects. Depends on the content scale, which can
            # potentially be dynamically modified, so we re-adjust the size limits every
            # time here.
            min_size = int(2 * icon_bar_width * g_pool.gui.scale /
                           framebuffer_scale)
            glfw.set_window_size_limits(
                window,
                min_size,
                min_size,
                glfw.DONT_CARE,
                glfw.DONT_CARE,
            )

            # Needed, to update the window buffer while resizing
            with gl_utils.current_context(main_window):
                consume_events_and_render_buffer()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
                window, x, y, cached_scale=None)
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, paths):
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_world"))
        if parse_version(session_settings.get("version",
                                              "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.min_data_confidence = 0.6
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", 0.8)
        g_pool.pupil_detection_enabled = bool(
            session_settings.get("pupil_detection_enabled", True))
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.set_audio_mode(
            session_settings.get("audio_mode", audio.get_default_audio_mode()))

        def handle_notifications(noti):
            subject = noti["subject"]
            if subject == "pupil_detector.set_enabled":
                g_pool.pupil_detection_enabled = noti["value"]
            elif subject == "start_plugin":
                try:
                    g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]],
                                       args=noti.get("args", {}))
                except KeyError as err:
                    logger.error(f"Attempt to load unknown plugin: {err}")
            elif subject == "stop_plugin":
                for p in g_pool.plugins:
                    if p.class_name == noti["name"]:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == "eye_process.started":
                noti = {
                    "subject": "pupil_detector.set_enabled",
                    "value": g_pool.pupil_detection_enabled,
                }
                ipc_pub.notify(noti)
            elif subject == "set_min_calibration_confidence":
                g_pool.min_calibration_confidence = noti["value"]
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify({
                    "subject": "meta.doc",
                    "actor": g_pool.app,
                    "doc": world.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            "subject": "meta.doc",
                            "actor": p.class_name,
                            "doc": p.on_notify.__doc__,
                        })
            elif subject == "world_process.adapt_window_size":
                set_window_size()
            elif subject == "world_process.should_stop":
                glfw.set_window_should_close(main_window, True)

        width, height = session_settings.get("window_size",
                                             (1280 + icon_bar_width, 720))

        # window and gl setup
        glfw.init()
        glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
        if hide_ui:
            glfw.window_hint(glfw.VISIBLE, 0)  # hide window
        main_window = glfw.create_window(width, height,
                                         "Pupil Capture - World", None, None)

        window_position_manager = gl_utils.WindowPositionManager()
        window_pos = window_position_manager.new_window_position(
            window=main_window,
            default_position=window_position_default,
            previous_position=session_settings.get("window_position", None),
        )
        glfw.set_window_pos(main_window, window_pos[0], window_pos[1])

        glfw.make_context_current(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.set_window_should_close(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({
                "subject": "world_process.should_start",
                "delay": 2.0
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is opened, the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-400, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
                                             (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")

        def set_window_size():
            # Get current capture frame size
            f_width, f_height = g_pool.capture.frame_size

            # Get current display scale factor
            content_scale = gl_utils.get_content_scale(main_window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
            display_scale_factor = content_scale / framebuffer_scale

            # Scale the capture frame size by display scale factor
            f_width *= display_scale_factor
            f_height *= display_scale_factor

            # Increas the width to account for the added scaled icon bar width
            f_width += icon_bar_width * display_scale_factor

            # Set the newly calculated size (scaled capture frame size + scaled icon bar width)
            glfw.set_window_size(main_window, int(f_width), int(f_height))

        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Selector(
                "Audio mode",
                None,
                getter=audio.get_audio_mode,
                setter=audio.set_audio_mode,
                selection=audio.get_audio_mode_list(),
            ))

        general_settings.append(
            ui.Switch(
                "pupil_detection_enabled",
                label="Pupil detection",
                getter=detection_enabled_getter,
                setter=detection_enabled_setter,
            ))
        general_settings.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: start_stop_eye(0, alive),
                getter=lambda: eye_procs_alive[0].value,
            ))
        general_settings.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: start_stop_eye(1, alive),
                getter=lambda: eye_procs_alive[1].value,
            ))

        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(g_pool.version)))
        general_settings.append(
            ui.Button("Restart with default settings", reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        loaded_plugins = session_settings.get("loaded_plugins",
                                              default_plugins)

        # Resolve the active calibration choreography plugin
        loaded_plugins = patch_loaded_plugins_with_choreography_plugin(
            loaded_plugins, app=g_pool.app)
        session_settings["loaded_plugins"] = loaded_plugins

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(g_pool, loaded_plugins)

        if not g_pool.capture:
            # Make sure we always have a capture running. Important if there was no
            # capture stored in session settings.
            g_pool.plugins.add(g_pool.plugin_by_name[default_capture_name],
                               default_capture_settings)

        # Register callbacks main_window
        glfw.set_framebuffer_size_callback(main_window, on_resize)
        glfw.set_key_callback(main_window, on_window_key)
        glfw.set_char_callback(main_window, on_window_char)
        glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
        glfw.set_cursor_pos_callback(main_window, on_pos)
        glfw.set_scroll_callback(main_window, on_scroll)
        glfw.set_drop_callback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(True)

        # now that we have a proper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})
        # If previously selected plugin was not loaded this time, we will have an
        # expanded menubar without any menu selected. We need to ensure the menubar is
        # collapsed in this case.
        if all(submenu.collapsed for submenu in g_pool.menubar.elements):
            g_pool.menubar.collapsed = True

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        g_pool.trigger_main_window_redraw = lambda: on_resize(
            main_window, *glfw.get_framebuffer_size(main_window))
        g_pool.trigger_main_window_redraw()

        if session_settings.get("eye1_process_alive", True):
            launch_eye_process(1, delay=0.6)
        if session_settings.get("eye0_process_alive", True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({"subject": "world_process.started"})
        logger.warning("Process started.")

        if platform.system() == "Darwin":
            # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of
            # occluded windows, based on the swap interval value. This causes an FPS drop
            # and leads to problems when recording. To side-step this behaviour, the swap
            # interval is set to zero.
            #
            # Read more about window occlusion on macOS here:
            # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html
            glfw.swap_interval(0)

        # Event loop
        while not glfw.window_should_close(
                main_window) and not process_was_interrupted:

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # "blacklisted" events that were already sent
            del events["pupil"]
            del events["gaze"]
            # delete if exists. More expensive than del, so only use it when key might not exist
            events.pop("annotation", None)

            # send new events to ipc:
            if "frame" in events:
                del events["frame"]  # send explicitly with frame publisher
            if "depth_frame" in events:
                del events["depth_frame"]
            if "audio_packets" in events:
                del events["audio_packets"]
            del events["dt"]  # no need to send this
            for data in events.values():
                assert isinstance(data, (list, tuple))
                for d in data:
                    ipc_pub.send(d)

            glfw.make_context_current(main_window)
            # render visual feedback from loaded plugins
            glfw.poll_events()
            if window_should_update() and gl_utils.is_window_visible(
                    main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                try:
                    clipboard = glfw.get_clipboard_string(main_window).decode()
                except (AttributeError, glfw.GLFWError):
                    # clipboard is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.set_clipboard_string(main_window,
                                              user_input.clipboard)

                for button, action, mods in user_input.buttons:
                    x, y = glfw.get_cursor_pos(main_window)
                    pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                        main_window, x, y, cached_scale=None)
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                glfw.swap_buffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)
        session_settings["eye0_process_alive"] = eye_procs_alive[0].value
        session_settings["eye1_process_alive"] = eye_procs_alive[1].value
        session_settings[
            "min_calibration_confidence"] = g_pool.min_calibration_confidence
        session_settings[
            "pupil_detection_enabled"] = g_pool.pupil_detection_enabled
        session_settings["audio_mode"] = audio.get_audio_mode()

        if not hide_ui:
            glfw.restore_window(main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.get_window_pos(
                main_window)
            session_window_size = glfw.get_window_size(main_window)
            if 0 not in session_window_size:
                f_width, f_height = session_window_size
                if platform.system() in ("Windows", "Linux"):
                    f_width, f_height = (
                        f_width / content_scale,
                        f_height / content_scale,
                    )
                session_settings["window_size"] = int(f_width), int(f_height)

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.destroy_window(main_window)
        glfw.terminate()

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Capture crashed with trace:\n{}".format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "world_process.stopped"})
        sleep(1.0)
    def recent_events(self, events):
        frame = events.get('frame')
        if not frame:
            return
        img = frame.img
        img_shape = img.shape[:-1][::-1]  # width,height

        succeeding_frame = frame.index - self.prev_frame_idx == 1
        same_frame = frame.index == self.prev_frame_idx
        gray_img = frame.gray

        #vars for calcOpticalFlowPyrLK
        lk_params = dict(winSize=(90, 90),
                         maxLevel=3,
                         criteria=(cv2.TERM_CRITERIA_EPS
                                   | cv2.TERM_CRITERIA_COUNT, 20, 0.03))

        updated_past_gaze = []

        #lets update past gaze using optical flow: this is like sticking the gaze points onto the pixels of the img.
        if self.past_gaze_positions and succeeding_frame:
            past_screen_gaze = np.array([
                denormalize(ng['norm_pos'], img_shape, flip_y=True)
                for ng in self.past_gaze_positions
            ],
                                        dtype=np.float32)
            new_pts, status, err = cv2.calcOpticalFlowPyrLK(
                self.prev_gray,
                gray_img,
                past_screen_gaze,
                None,
                minEigThreshold=0.005,
                **lk_params)
            for gaze, new_gaze_pt, s, e in zip(self.past_gaze_positions,
                                               new_pts, status, err):
                if s:
                    # print "norm,updated",gaze['norm_gaze'], normalize(new_gaze_pt,img_shape[:-1],flip_y=True)
                    gaze['norm_pos'] = normalize(new_gaze_pt,
                                                 img_shape,
                                                 flip_y=True)
                    updated_past_gaze.append(gaze)
                    # logger.debug("updated gaze")

                else:
                    # logger.debug("dropping gaze")
                    # Since we will replace self.past_gaze_positions later,
                    # not appedning tu updated_past_gaze is like deliting this data point.
                    pass
        else:
            # we must be seeking, do not try to do optical flow, or pausing: see below.
            pass

        if same_frame and not self.gaze_changed:
            # paused
            # re-use last result
            events['gaze_positions'] = self.past_gaze_positions[:]
        else:
            # trim gaze that is too old
            if events['gaze_positions']:
                now = events['gaze_positions'][0]['timestamp']
                cutoff = now - self.timeframe
                updated_past_gaze = [
                    g for g in updated_past_gaze if g['timestamp'] > cutoff
                ]

            #inject the scan path gaze points into recent_gaze_positions
            events['gaze_positions'] = updated_past_gaze + events[
                'gaze_positions']
            events['gaze_positions'].sort(
                key=lambda x: x['timestamp'])  #this may be redundant...

        #update info for next frame.
        self.gaze_changed = False
        self.prev_gray = gray_img
        self.prev_frame_idx = frame.index
        self.past_gaze_positions = events['gaze_positions']
Example #53
0
    def recent_events(self, events):
        self.recent_fixation = None
        events['fixations'] = []
        fs = self.g_pool.capture.frame_size
        gaze = events['gaze_positions']

        self.queue.extend((gp for gp in gaze
                           if gp['confidence'] > self.confidence_threshold))

        try:  # use newest gaze point to determine age threshold
            age_threshold = self.queue[-1]['timestamp'] - self.duration / 1000.
            while self.queue[1]['timestamp'] < age_threshold:
                self.queue.popleft()  # remove outdated gaze points
        except IndexError:
            pass

        gaze_3d = [
            gp for gp in self.queue if '3d' in gp['base_data'][0]['method']
        ]
        use_3d = len(gaze_3d) > 0.8 * len(self.queue)

        if use_3d:
            base_data = gaze_3d
            points0 = [(pp['theta'], pp['phi'])
                       for pp in chain(*(gp['base_data'] for gp in gaze_3d))
                       if pp['id'] == 0]
            points1 = [(pp['theta'], pp['phi'])
                       for pp in chain(*(gp['base_data'] for gp in gaze_3d))
                       if pp['id'] == 1]
            points = np.array(
                points0 if len(points0) > len(points1) else points1,
                dtype=np.float32)
        else:
            base_data = list(self.queue)
            points = np.array(
                [denormalize(gp['norm_pos'], fs) for gp in base_data],
                dtype=np.float32)

        if points.shape[0] <= 2 or base_data[-1]['timestamp'] - base_data[0][
                'timestamp'] < self.duration / 1000.:
            self.recent_fixation = None
            return

        center, radius = cv2.minEnclosingCircle(points)
        radius *= 2  # all dispersion measures use the diameter instead of radius
        if use_3d and radius < np.deg2rad(self.dispersion_3d):
            norm_pos = np.mean([gp['norm_pos'] for gp in base_data],
                               axis=0).tolist()
            method = '3d pupil angle [deg]'
            dispersion = np.rad2deg(radius)  # in degrees

        elif not use_3d and radius < self.dispersion_2d:
            norm_pos = normalize(center, fs)
            method = '2d gaze [px]'
            dispersion = radius
        else:
            base_data = None

        if base_data:
            new_fixation = {
                'topic':
                'fixation',
                'norm_pos':
                norm_pos,
                'dispersion':
                dispersion,
                'method':
                method,
                'base_data':
                base_data,
                'timestamp':
                base_data[0]['timestamp'],
                'duration':
                base_data[-1]['timestamp'] - base_data[0]['timestamp'],
                'confidence':
                float(np.mean([gp['confidence'] for gp in base_data]))
            }
            events['fixations'].append(new_fixation)
            self.recent_fixation = new_fixation
        else:
            self.recent_fixation = None
    def update(self, frame, events):
        for eye_index in self.showeyes:
            requested_eye_frame_idx = self.eye_world_frame_map[eye_index][
                frame.index]

            #1. do we need a new frame?
            if requested_eye_frame_idx != self.eye_frames[eye_index].index:
                # do we need to seek?
                if requested_eye_frame_idx == self.eye_cap[
                        eye_index].get_frame_index() + 1:
                    # if we just need to seek by one frame, its faster to just read one and and throw it away.
                    _ = self.eye_cap[eye_index].get_frame()
                if requested_eye_frame_idx != self.eye_cap[
                        eye_index].get_frame_index():
                    # only now do I need to seek
                    self.eye_cap[eye_index].seek_to_frame(
                        requested_eye_frame_idx)
                # reading the new eye frame frame
                try:
                    self.eye_frames[eye_index] = self.eye_cap[
                        eye_index].get_frame()
                except EndofVideoFileError:
                    logger.warning(
                        "Reached the end of the eye video for eye video {}.".
                        format(eye_index))
            else:
                #our old frame is still valid because we are doing upsampling
                pass

            #2. dragging image
            if self.drag_offset[eye_index] is not None:
                pos = glfwGetCursorPos(glfwGetCurrentContext())
                pos = normalize(pos,
                                glfwGetWindowSize(glfwGetCurrentContext()))
                pos = denormalize(
                    pos, (frame.img.shape[1],
                          frame.img.shape[0]))  # Position in img pixels
                self.pos[eye_index][
                    0] = pos[0] + self.drag_offset[eye_index][0]
                self.pos[eye_index][
                    1] = pos[1] + self.drag_offset[eye_index][1]
            else:
                self.video_size = [
                    round(self.eye_frames[eye_index].width *
                          self.eye_scale_factor),
                    round(self.eye_frames[eye_index].height *
                          self.eye_scale_factor)
                ]

            #3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
            self.pos[eye_index][1] = min(
                frame.img.shape[0] - self.video_size[1],
                max(self.pos[eye_index][1], 0)
            )  #frame.img.shape[0] is height, frame.img.shape[1] is width of screen
            self.pos[eye_index][0] = min(
                frame.img.shape[1] - self.video_size[0],
                max(self.pos[eye_index][0], 0))

            #4. flipping images, converting to greyscale
            eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,
                                    cv2.COLOR_BGR2GRAY)  #auto gray scaling
            eyeimage = cv2.resize(eye_gray, (0, 0),
                                  fx=self.eye_scale_factor,
                                  fy=self.eye_scale_factor)
            if self.mirror[str(eye_index)]:
                eyeimage = np.fliplr(eyeimage)
            if self.flip[str(eye_index)]:
                eyeimage = np.flipud(eyeimage)

            eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)

            if self.show_ellipses and events['pupil_positions']:
                for pd in events['pupil_positions']:
                    if pd['id'] == eye_index and pd[
                            'timestamp'] == self.eye_frames[
                                eye_index].timestamp:
                        break

                el = pd['ellipse']
                conf = int(
                    pd.get('model_confidence', pd.get('confidence', 0.1)) *
                    255)
                center = list(
                    map(lambda val: int(self.eye_scale_factor * val),
                        el['center']))
                el['axes'] = tuple(
                    map(lambda val: int(self.eye_scale_factor * val / 2),
                        el['axes']))
                el['angle'] = int(el['angle'])
                el_points = cv2.ellipse2Poly(tuple(center), el['axes'],
                                             el['angle'], 0, 360, 1)

                if self.mirror[str(eye_index)]:
                    el_points = [(self.video_size[0] - x, y)
                                 for x, y in el_points]
                    center[0] = self.video_size[0] - center[0]
                if self.flip[str(eye_index)]:
                    el_points = [(x, self.video_size[1] - y)
                                 for x, y in el_points]
                    center[1] = self.video_size[1] - center[1]

                cv2.polylines(eyeimage, [np.asarray(el_points)],
                              True, (0, 0, 255, conf),
                              thickness=math.ceil(2 * self.eye_scale_factor))
                cv2.circle(eyeimage,
                           tuple(center),
                           int(5 * self.eye_scale_factor), (0, 0, 255, conf),
                           thickness=-1)

            # 5. finally overlay the image
            x, y = int(self.pos[eye_index][0]), int(self.pos[eye_index][1])
            transparent_image_overlay((x, y), eyeimage, frame.img, self.alpha)
Example #55
0
def player(
    rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version, debug
):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime

    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        from OpenGL.GL import GL_COLOR_BUFFER_BIT

        # display
        import glfw

        glfw.ERROR_REPORTING = "raise"

        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils

        # capture
        from video_capture import File_Source

        # helpers/utils
        from version_utils import parse_version
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from pupil_recording import PupilRecording
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation

        from seek_control import Seek_Control
        from surface_tracker import Surface_Tracker_Offline

        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producer.gaze_from_recording import GazeFromRecording
        from gaze_producer.gaze_from_offline_calibration import (
            GazeFromOfflineCalibration,
        )
        from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from video_export.plugins.imotions_exporter import iMotions_Exporter
        from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
        from video_export.plugins.world_video_exporter import World_Video_Exporter
        from head_pose_tracker.offline_head_pose_tracker import (
            Offline_Head_Pose_Tracker,
        )
        from video_capture import File_Source
        from video_overlay.plugins import Video_Overlay, Eye_Overlay

        from pupil_recording import (
            assert_valid_recording_type,
            InvalidRecordingException,
        )

        assert parse_version(pyglui_version) >= parse_version(
            "1.28"
        ), "pyglui out of date, please upgrade to newest version"

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins"))
        runtime_plugins = [
            p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin)
        ]
        system_plugins = [
            Log_Display,
            Seek_Control,
            Plugin_Manager,
            System_Graphs,
            System_Timelines,
            Audio_Playback,
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Eye_Overlay,
            Video_Overlay,
            Offline_Fixation_Detector,
            Offline_Blink_Detection,
            Surface_Tracker_Offline,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            GazeFromRecording,
            GazeFromOfflineCalibration,
            World_Video_Exporter,
            iMotions_Exporter,
            Eye_Video_Exporter,
            Offline_Head_Pose_Tracker,
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        def consume_events_and_render_buffer():
            gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
            g_pool.capture.gl_display()
            for p in g_pool.plugins:
                p.gl_display()

            gl_utils.glViewport(0, 0, *window_size)

            try:
                clipboard = glfw.get_clipboard_string(main_window).decode()
            except (AttributeError, glfw.GLFWError):
                # clipbaord is None, might happen on startup
                clipboard = ""
            g_pool.gui.update_clipboard(clipboard)
            user_input = g_pool.gui.update()
            if user_input.clipboard and user_input.clipboard != clipboard:
                # only write to clipboard if content changed
                glfw.set_clipboard_string(main_window, user_input.clipboard)

            for b in user_input.buttons:
                button, action, mods = b
                x, y = glfw.get_cursor_pos(main_window)
                pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                    main_window, x, y, cached_scale=None
                )
                pos = normalize(pos, g_pool.camera_render_size)
                pos = denormalize(pos, g_pool.capture.frame_size)

                for plugin in g_pool.plugins:
                    if plugin.on_click(pos, button, action):
                        break

            for key, scancode, action, mods in user_input.keys:
                for plugin in g_pool.plugins:
                    if plugin.on_key(key, scancode, action, mods):
                        break

            for char_ in user_input.chars:
                for plugin in g_pool.plugins:
                    if plugin.on_char(char_):
                        break

            glfw.swap_buffers(main_window)

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal content_scale
            if w == 0 or h == 0:
                return

            # Always clear buffers on resize to make sure that there are no overlapping
            # artifacts from previous frames.
            gl_utils.glClear(GL_COLOR_BUFFER_BIT)
            gl_utils.glClearColor(0, 0, 0, 1)

            content_scale = gl_utils.get_content_scale(window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(window)
            g_pool.gui.scale = content_scale
            window_size = w, h
            g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

            # Minimum window size required, otherwise parts of the UI can cause openGL
            # issues with permanent effects. Depends on the content scale, which can
            # potentially be dynamically modified, so we re-adjust the size limits every
            # time here.
            min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale)
            glfw.set_window_size_limits(
                window,
                min_size,
                min_size,
                glfw.DONT_CARE,
                glfw.DONT_CARE,
            )

            # Needed, to update the window buffer while resizing
            consume_events_and_render_buffer()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
                window, x, y, cached_scale=None
            )
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, paths):
            for path in paths:
                try:
                    assert_valid_recording_type(path)
                    _restart_with_recording(path)
                    return
                except InvalidRecordingException as err:
                    logger.debug(str(err))

            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        def _restart_with_recording(rec_dir):
            logger.debug("Starting new session with '{}'".format(rec_dir))
            ipc_pub.notify(
                {"subject": "player_drop_process.should_start", "rec_dir": rec_dir}
            )
            glfw.set_window_should_close(g_pool.main_window, True)

        tick = delta_t()

        def get_dt():
            return next(tick)

        recording = PupilRecording(rec_dir)
        meta_info = recording.meta_info

        # log info about Pupil Platform and Platform in player.log
        logger.info("Application Version: {}".format(app_version))
        logger.info("System Info: {}".format(get_system_info()))
        logger.debug(f"Debug flag: {debug}")

        icon_bar_width = 50
        window_size = None
        content_scale = 1.0

        # create container for globally scoped vars
        g_pool = SimpleNamespace()
        g_pool.app = "player"
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        video_path = recording.files().core().world().videos()[0].resolve()
        File_Source(
            g_pool,
            timing="external",
            source_path=video_path,
            buffered_decoding=True,
            fill_gaps=True,
        )

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player")
        )
        if parse_version(session_settings.get("version", "0.0")) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}"

        glfw.init()
        glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
        main_window = glfw.create_window(width, height, window_name, None, None)

        window_position_manager = gl_utils.WindowPositionManager()
        window_pos = window_position_manager.new_window_position(
            window=main_window,
            default_position=window_position_default,
            previous_position=session_settings.get("window_position", None),
        )
        glfw.set_window_pos(main_window, window_pos[0], window_pos[1])

        glfw.make_context_current(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.0
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT
        )
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT
        )

        # populated by producers
        g_pool.pupil_positions = pm.PupilDataBisector()
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()
        g_pool.eye_movements = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {"subject": "min_data_confidence_changed"}
            notification["_notify_time_"] = time() + 0.8
            g_pool.ipc_pub.notify(notification)

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop
            export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx))

            export_dir = os.path.join(g_pool.rec_dir, "exports")
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                "Player Software Version": str(g_pool.version),
                "Data Format Version": meta_info.min_player_version,
                "Export Date": strftime("%d.%m.%Y", localtime()),
                "Export Time": strftime("%H:%M:%S", localtime()),
                "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(),
                "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(),
                "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(),
            }
            with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
                write_key_value_file(csv, export_info)

            notification = {
                "subject": "should_export",
                "range": export_range,
                "ts_window": export_ts_window,
                "export_dir": export_dir,
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.set_window_should_close(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify(
                {
                    "subject": "player_process.should_start",
                    "rec_dir": rec_dir,
                    "delay": 2.0,
                }
            )

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu(
            "User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline"
        )
        g_pool.user_timelines.color = RGBA(a=0.0)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            # Get current capture frame size
            f_width, f_height = g_pool.capture.frame_size

            # Get current display scale factor
            content_scale = gl_utils.get_content_scale(main_window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
            display_scale_factor = content_scale / framebuffer_scale

            # Scale the capture frame size by display scale factor
            f_width *= display_scale_factor
            f_height *= display_scale_factor

            # Increas the width to account for the added scaled icon bar width
            f_width += icon_bar_width * display_scale_factor

            # Set the newly calculated size (scaled capture frame size + scaled icon bar width)
            glfw.set_window_size(main_window, int(f_width), int(f_height))

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(
            ui.Info_Text(f"Minimum Player Version: {meta_info.min_player_version}")
        )
        general_settings.append(ui.Info_Text(f"Player Version: {g_pool.version}"))
        general_settings.append(
            ui.Info_Text(f"Recording Software: {meta_info.recording_software_name}")
        )
        general_settings.append(
            ui.Info_Text(
                f"Recording Software Version: {meta_info.recording_software_version}"
            )
        )

        general_settings.append(
            ui.Info_Text(
                "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
            )
        )
        general_settings.append(
            ui.Slider(
                "min_data_confidence",
                g_pool,
                setter=set_data_confidence,
                step=0.05,
                min=0.0,
                max=1.0,
                label="Minimum data confidence",
            )
        )

        general_settings.append(
            ui.Button("Restart with default settings", reset_restart)
        )

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100))
        g_pool.export_button = ui.Thumb(
            "export",
            label=chr(0xE2C5),
            getter=lambda: False,
            setter=do_export,
            hotkey="e",
            label_font="pupil_icons",
        )
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [
            ("Plugin_Manager", {}),
            ("Seek_Control", {}),
            ("Log_Display", {}),
            ("Raw_Data_Exporter", {}),
            ("Vis_Polyline", {}),
            ("Vis_Circle", {}),
            ("System_Graphs", {}),
            ("System_Timelines", {}),
            ("World_Video_Exporter", {}),
            ("Pupil_From_Recording", {}),
            ("GazeFromRecording", {}),
            ("Audio_Playback", {}),
        ]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins)
        )

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                "rel_time_trim_section",
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label="Relative time range to export",
            ),
        )
        general_settings.insert(
            -1,
            ui.Text_Input(
                "frame_idx_trim_section",
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label="Frame index range to export",
            ),
        )

        # Register callbacks main_window
        glfw.set_framebuffer_size_callback(main_window, on_resize)
        glfw.set_key_callback(main_window, on_window_key)
        glfw.set_char_callback(main_window, on_window_char)
        glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
        glfw.set_cursor_pos_callback(main_window, on_pos)
        glfw.set_scroll_callback(main_window, on_scroll)
        glfw.set_drop_callback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get("ui_config", {})
        # If previously selected plugin was not loaded this time, we will have an
        # expanded menubar without any menu selected. We need to ensure the menubar is
        # collapsed in this case.
        if all(submenu.collapsed for submenu in g_pool.menubar.elements):
            g_pool.menubar.collapsed = True

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.get_framebuffer_size(main_window))

        def handle_notifications(n):
            subject = n["subject"]
            if subject == "start_plugin":
                g_pool.plugins.add(
                    g_pool.plugin_by_name[n["name"]], args=n.get("args", {})
                )
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify(
                    {"subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__}
                )
                for p in g_pool.plugins:
                    if (
                        p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify
                    ):
                        ipc_pub.notify(
                            {
                                "subject": "meta.doc",
                                "actor": p.class_name,
                                "doc": p.on_notify.__doc__,
                            }
                        )

        while not glfw.window_should_close(main_window) and not process_was_interrupted:

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events["pupil"] = []
            events["gaze"] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.make_context_current(main_window)
            glfw.poll_events()
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.get_clipboard_string(main_window).decode()
                except (AttributeError, glfw.GLFWError):
                    # clipbaord is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.set_clipboard_string(main_window, user_input.clipboard)

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.get_cursor_pos(main_window)
                    pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                        main_window, x, y, cached_scale=None
                    )
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                # present frames at appropriate speed
                g_pool.seek_control.wait(events["frame"].timestamp)
                glfw.swap_buffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["min_data_confidence"] = g_pool.min_data_confidence
        session_settings[
            "min_calibration_confidence"
        ] = g_pool.min_calibration_confidence
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["window_position"] = glfw.get_window_pos(main_window)
        session_settings["version"] = str(g_pool.version)

        session_window_size = glfw.get_window_size(main_window)
        if 0 not in session_window_size:
            f_width, f_height = session_window_size
            if platform.system() in ("Windows", "Linux"):
                f_width, f_height = (
                    f_width / content_scale,
                    f_height / content_scale,
                )
            session_settings["window_size"] = int(f_width), int(f_height)

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.destroy_window(main_window)

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Player crashed with trace:\n{}".format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "player_process.stopped"})
        sleep(1.0)
Example #56
0
def player(rec_dir, ipc_pub_url, ipc_sub_url,
           ipc_push_url, user_dir, app_version):
    # general imports
    from time import sleep
    import logging
    import errno
    from glob import glob
    from copy import deepcopy
    from time import time
    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        # imports
        from file_methods import Persistent_Dict, load_object

        # display
        import glfw
        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils
        # capture
        from video_capture import File_Source, EndofVideoFileError

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation
        from vis_scan_path import Vis_Scan_Path
        from vis_eye_video_overlay import Vis_Eye_Video_Overlay
        from seek_control import Seek_Control
        from video_export_launcher import Video_Export_Launcher
        from offline_surface_tracker import Offline_Surface_Tracker
        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from batch_exporter import Batch_Exporter, Batch_Export
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producers import Gaze_From_Recording, Offline_Calibration
        from system_graphs import System_Graphs

        assert VersionFormat(pyglui_version) >= VersionFormat('1.13'), 'pyglui out of date, please upgrade to newest version'

        runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
        system_plugins = [Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export]
        user_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points,
                        Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path,
                        Offline_Fixation_Detector, Batch_Exporter,
                        Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter,
                        Annotation_Player, Log_History,
                        Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording,
                        Offline_Calibration] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor

            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y*scroll_factor)

        def on_drop(window, count, paths):
            for x in range(count):
                new_rec_dir = paths[x].decode('utf-8')
                if is_pupil_rec_dir(new_rec_dir):
                    logger.debug("Starting new session with '{}'".format(new_rec_dir))
                    ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
                    glfw.glfwSetWindowShouldClose(window, True)
                else:
                    logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

        tick = delta_t()

        def get_dt():
            return next(tick)

        video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
                      if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
        pupil_data_path = os.path.join(rec_dir, "pupil_data")

        meta_info = load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info('Application Version: {}'.format(app_version))
        logger.info('System Info: {}'.format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = Global_Container()
        g_pool.app = 'player'
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        # sets itself to g_pool.capture
        File_Source(g_pool, video_path)

        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version", '0.0')) != app_version:
            logger.info("Session setting are a different version of this app. I will not use those.")
            session_settings.clear()

        width, height = session_settings.get('window_size', g_pool.capture.frame_size)
        window_pos = session_settings.get('window_position', window_position_default)
        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                       + rec_dir.split(os.path.sep)[-1], None, None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (g_pool.camera_render_size[0] + int(icon_bar_width*g_pool.gui_user_scale*hdpi_factor),
                           glfw.glfwGetFramebufferSize(main_window)[1])
            logger.warning(icon_bar_width*g_pool.gui_user_scale*hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.pupil_data = load_object(pupil_data_path)
        g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.
        g_pool.new_seek = True
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)

        g_pool.pupil_positions = []
        g_pool.gaze_positions = []
        g_pool.fixations = []

        g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
        g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps]  # populated by producer`
        g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps]  # populated by producer
        g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

        # def next_frame(_):
        #     try:
        #         g_pool.capture.seek_to_frame(g_pool.capture.get_frame_index() + 1)
        #     except(FileSeekError):
        #         logger.warning("Could not seek to next frame.")
        #     else:
        #         g_pool.new_seek = True

        # def prev_frame(_):
        #     try:
        #         g_pool.capture.seek_to_frame(g_pool.capture.get_frame_index() - 1)
        #     except(FileSeekError):
        #         logger.warning("Could not seek to previous frame.")
        #     else:
        #         g_pool.new_seek = True

        # def toggle_play(new_state):
        #     if g_pool.capture.get_frame_index() >= g_pool.capture.get_frame_count()-5:
        #         g_pool.capture.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
        #         logger.warning("End of video - restart at beginning.")
        #     g_pool.capture.play = new_state

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {'subject': 'min_data_confidence_changed'}
            notification['_notify_time_'] = time()+.8
            g_pool.ipc_pub.notify(notification)

        def open_plugin(plugin):
            if plugin == "Select to load":
                return
            g_pool.plugins.add(plugin)

        def purge_plugins():
            for p in g_pool.plugins:
                if p.__class__ in user_plugins:
                    p.alive = False
            g_pool.plugins.clean()

        def do_export(_):
            export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right
            export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
            try:
                os.makedirs(export_dir)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    logger.error("Could not create export dir")
                    raise e
                else:
                    overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                    logger.warning(overwrite_warning.format(*export_range))
            else:
                logger.info('Created export dir at "{}"'.format(export_dir))

            notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2.})

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width,0),size=(0,0),header_pos='hidden')
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.),
                                                 size=(0., 0.), header_pos='headline')
        g_pool.user_timelines.color = RGBA(a=0.)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(ui.Button('Reset window size',
                                          lambda: glfw.glfwSetWindowSize(main_window, g_pool.capture.frame_size[0], g_pool.capture.frame_size[1])))
        general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2]+list(np.arange(1.5, 5.1, .5)), label='Interface Size'))
        general_settings.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
        general_settings.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
        general_settings.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
        general_settings.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                          step=.05, min=0.0, max=1.0, label='Confidence threshold'))
        general_settings.append(ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100))
        g_pool.export_button = ui.Thumb('export',
                                        label=chr(0xe2c4),
                                        getter=lambda: False,
                                        setter=do_export,
                                        hotkey='e',
                                        label_font='pupil_icons')
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}),
                           ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}),
                           ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
        g_pool.plugins = Plugin_List(g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'start_plugin':
                g_pool.plugins.add(
                    g_pool.plugin_by_name[n['name']], args=n.get('args', {}))
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({'subject': 'meta.doc',
                                'actor': g_pool.app,
                                'doc': player.__doc__})
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({'subject': 'meta.doc',
                                        'actor': p.class_name,
                                        'doc': p.on_notify.__doc__})

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # grab new frame
            if g_pool.capture.play or g_pool.new_seek:
                g_pool.new_seek = False
                try:
                    new_frame = g_pool.capture.get_frame()
                except EndofVideoFileError:
                    # end of video logic: pause at last frame.
                    g_pool.capture.play = False
                    logger.warning("end of video")

            frame = new_frame.copy()
            events = {}
            events['frame'] = frame
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events['pupil_positions'] = []
            events['gaze_positions'] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture._recent_frame = frame
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                unused_elements = g_pool.gui.update()
                for b in unused_elements.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in unused_elements.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in unused_elements.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                glfw.glfwSwapBuffers(main_window)

            # present frames at appropriate speed
            g_pool.capture.wait(frame)
            glfw.glfwPollEvents()

        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['min_data_confidence'] = g_pool.min_data_confidence
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.capture.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Player crashed with trace:\n{}'.format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'player_process.stopped'})
        sleep(1.0)
Example #57
0
    def update(self, frame, events):
        self.img_shape = frame.height, frame.width, 3

        if self.running:
            gray = frame.gray
            if self.invert_image:
                gray = 255 - gray

            if self.robust_detection:
                self.markers = detect_markers_robust(
                    gray,
                    grid_size=5,
                    aperture=self.aperture,
                    prev_markers=self.markers,
                    true_detect_every_frame=3,
                    min_marker_perimeter=self.min_marker_perimeter)
            else:
                self.markers = detect_markers(
                    gray,
                    grid_size=5,
                    aperture=self.aperture,
                    min_marker_perimeter=self.min_marker_perimeter)
            if self.mode == "Show marker IDs":
                draw_markers(frame.gray, self.markers)

        # locate surfaces, map gaze
        for s in self.surfaces:
            s.locate(self.markers, self.camera_calibration,
                     self.min_marker_perimeter, self.min_id_confidence,
                     self.locate_3d)
            if s.detected:
                s.gaze_on_srf = s.map_data_to_surface(
                    events.get('gaze_positions', []), s.m_from_screen)
            else:
                s.gaze_on_srf = []

        events['surface'] = []
        for s in self.surfaces:
            if s.detected:
                events['surface'].append({
                    'name':
                    s.name,
                    'uid':
                    s.uid,
                    'm_to_screen':
                    s.m_to_screen.tolist(),
                    'm_from_screen':
                    s.m_from_screen.tolist(),
                    'gaze_on_srf':
                    s.gaze_on_srf,
                    'timestamp':
                    frame.timestamp,
                    'camera_pose_3d':
                    s.camera_pose_3d.tolist()
                    if s.camera_pose_3d is not None else None
                })

        if self.running:
            self.button.status_text = '{}/{}'.format(
                len([s for s in self.surfaces if s.detected]),
                len(self.surfaces))
        else:
            self.button.status_text = 'tracking paused'

        if self.mode == 'Show Markers and Surfaces':
            # edit surfaces by user
            if self.edit_surf_verts:
                window = glfwGetCurrentContext()
                pos = glfwGetCursorPos(window)
                pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
                for s, v_idx in self.edit_surf_verts:
                    if s.detected:
                        new_pos = s.img_to_ref_surface(np.array(pos))
                        s.move_vertex(v_idx, new_pos)
Example #58
0
    def update(self, frame, events):
        for eye_index in self.showeyes:
            requested_eye_frame_idx = self.eye_world_frame_map[eye_index][
                frame.index]

            #1. do we need a new frame?
            if requested_eye_frame_idx != self.eye_frames[eye_index].index:
                # do we need to seek?
                if requested_eye_frame_idx == self.eye_cap[
                        eye_index].get_frame_index() + 1:
                    # if we just need to seek by one frame, its faster to just read one and and throw it away.
                    _ = self.eye_cap[eye_index].get_frame()
                if requested_eye_frame_idx != self.eye_cap[
                        eye_index].get_frame_index():
                    # only now do I need to seek
                    self.eye_cap[eye_index].seek_to_frame(
                        requested_eye_frame_idx)
                # reading the new eye frame frame
                try:
                    self.eye_frames[eye_index] = self.eye_cap[
                        eye_index].get_frame()
                except EndofVideoFileError:
                    logger.warning(
                        "Reached the end of the eye video for eye video %s." %
                        eye_index)
            else:
                #our old frame is still valid because we are doing upsampling
                pass

            #2. dragging image
            if self.drag_offset[eye_index] is not None:
                pos = glfwGetCursorPos(glfwGetCurrentContext())
                pos = normalize(pos,
                                glfwGetWindowSize(glfwGetCurrentContext()))
                pos = denormalize(
                    pos, (frame.img.shape[1],
                          frame.img.shape[0]))  # Position in img pixels
                self.pos[eye_index][
                    0] = pos[0] + self.drag_offset[eye_index][0]
                self.pos[eye_index][
                    1] = pos[1] + self.drag_offset[eye_index][1]
            else:
                self.video_size = [
                    round(self.eye_frames[eye_index].width *
                          self.eye_scale_factor),
                    round(self.eye_frames[eye_index].height *
                          self.eye_scale_factor)
                ]

            #3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
            self.pos[eye_index][1] = min(
                frame.img.shape[0] - self.video_size[1],
                max(self.pos[eye_index][1], 0)
            )  #frame.img.shape[0] is height, frame.img.shape[1] is width of screen
            self.pos[eye_index][0] = min(
                frame.img.shape[1] - self.video_size[0],
                max(self.pos[eye_index][0], 0))

            #4. flipping images, converting to greyscale
            eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,
                                    cv2.COLOR_BGR2GRAY)  #auto gray scaling
            eyeimage = cv2.resize(eye_gray, (0, 0),
                                  fx=self.eye_scale_factor,
                                  fy=self.eye_scale_factor)
            if self.mirror[str(eye_index)]:
                eyeimage = np.fliplr(eyeimage)
            if self.flip[str(eye_index)]:
                eyeimage = np.flipud(eyeimage)

            #5. finally overlay the image
            x, y = int(self.pos[eye_index][0]), int(self.pos[eye_index][1])
            transparent_image_overlay((x, y),
                                      cv2.cvtColor(eyeimage,
                                                   cv2.COLOR_GRAY2BGR),
                                      frame.img, self.alpha)
Example #59
0
    def visualize(self, frame, alpha, scale, show_ellipses, pupil_positions):
        if not self.initialized:
            return

        requested_eye_frame_idx = self.eye_world_frame_map[frame.index]
        # 1. do we need a new frame?
        if requested_eye_frame_idx != self.current_eye_frame.index:
            if requested_eye_frame_idx == self.source.get_frame_index() + 2:
                # if we just need to seek by one frame, its faster to just read one and and throw it away.
                self.source.get_frame()
            if requested_eye_frame_idx != self.source.get_frame_index() + 1:
                self.source.seek_to_frame(requested_eye_frame_idx)

            try:
                self.current_eye_frame = self.source.get_frame()
            except EndofVideoFileError:
                logger.info("Reached the end of the eye video for eye video {}.".format(self.eyeid))

        # 2. dragging image
        if self.drag_offset is not None:
            pos = glfwGetCursorPos(glfwGetCurrentContext())
            pos = normalize(pos, glfwGetWindowSize(glfwGetCurrentContext()))
            # Position in img pixels
            pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))
            self.pos = int(pos[0] + self.drag_offset[0]), int(pos[1] + self.drag_offset[1])

        # 3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
        video_size = round(self.current_eye_frame.width * scale), round(self.current_eye_frame.height * scale)

        # frame.img.shape[0] is height, frame.img.shape[1] is width of screen
        self.pos = (min(frame.img.shape[1] - video_size[0], max(self.pos[0], 0)),
                    min(frame.img.shape[0] - video_size[1], max(self.pos[1], 0)))

        # 4. vflipping images, converting to greyscale
        eye_gray = self.current_eye_frame.gray
        eyeimage = cv2.resize(eye_gray, (0, 0), fx=scale, fy=scale)
        if self.hflip:
            eyeimage = np.fliplr(eyeimage)
        if self.vflip:
            eyeimage = np.flipud(eyeimage)

        eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)
        if show_ellipses:
            try:
                pp = next((pp for pp in pupil_positions if pp['id'] == self.eyeid and pp['timestamp'] == self.current_eye_frame.timestamp))
            except StopIteration:
                pass
            else:
                el = pp['ellipse']
                conf = int(pp.get('model_confidence', pp.get('confidence', 0.1)) * 255)
                center = list(map(lambda val: int(scale*val), el['center']))
                el['axes'] = tuple(map(lambda val: int(scale*val/2), el['axes']))
                el['angle'] = int(el['angle'])
                el_points = cv2.ellipse2Poly(tuple(center), el['axes'], el['angle'], 0, 360, 1)
                if self.hflip:
                    el_points = [(video_size[0] - x, y) for x, y in el_points]
                    center[0] = video_size[0] - center[0]
                if self.vflip:
                    el_points = [(x, video_size[1] - y) for x, y in el_points]
                    center[1] = video_size[1] - center[1]

                cv2.polylines(eyeimage, [np.asarray(el_points)], True, (0, 0, 255, conf), thickness=int(np.ceil(2*scale)))
                cv2.circle(eyeimage, tuple(center), int(5*scale), (0, 0, 255, conf), thickness=-1)

        transparent_image_overlay(self.pos, eyeimage, frame.img, alpha)
Example #60
0
    def _map_binocular(self, p0, p1):

        if '3d' not in p0['method'] or '3d' not in p1['method']:
            return None

        #find the nearest intersection point of the two gaze lines
        # a line is defined by two point
        s0_center = self.eye0_to_World(np.array(p0['sphere']['center']))
        s1_center = self.eye1_to_World(np.array(p1['sphere']['center']))

        s0_normal = np.dot(self.rotation_matricies[0],
                           np.array(p0['circle_3d']['normal']))
        s1_normal = np.dot(self.rotation_matricies[1],
                           np.array(p1['circle_3d']['normal']))

        gaze_line0 = [s0_center, s0_center + s0_normal]
        gaze_line1 = [s1_center, s1_center + s1_normal]

        nearest_intersection_point, intersection_distance = math_helper.nearest_intersection(
            gaze_line0, gaze_line1)

        if nearest_intersection_point is not None:
            self.last_gaze_distance = np.sqrt(
                nearest_intersection_point.dot(nearest_intersection_point))
            image_point, _ = cv2.projectPoints(
                np.array([nearest_intersection_point]),
                np.array([0.0, 0.0, 0.0]), np.array([0.0, 0.0, 0.0]),
                self.camera_matrix, self.dist_coefs)
            image_point = image_point.reshape(-1, 2)
            image_point = normalize(image_point[0],
                                    self.world_frame_size,
                                    flip_y=True)

        if self.visualizer.window:
            gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
            gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
            self.gaze_pts_debug0.append(gaze0_3d)
            self.gaze_pts_debug1.append(gaze1_3d)
            if nearest_intersection_point is not None:
                self.intersection_points_debug.append(
                    nearest_intersection_point)

            self.sphere0['center'] = s0_center  #eye camera coordinates
            self.sphere0['radius'] = p0['sphere']['radius']
            self.sphere1['center'] = s1_center  #eye camera coordinates
            self.sphere1['radius'] = p1['sphere']['radius']

        if nearest_intersection_point is None:
            return None

        confidence = (p0['confidence'] + p1['confidence']) / 2.
        ts = (p0['timestamp'] + p1['timestamp']) / 2.
        g = {
            'norm_pos': image_point,
            'eye_centers_3d': {
                0: s0_center.tolist(),
                1: s1_center.tolist()
            },
            'gaze_normals_3d': {
                0: s0_normal.tolist(),
                1: s1_normal.tolist()
            },
            'gaze_point_3d': nearest_intersection_point.tolist(),
            'confidence': confidence,
            'timestamp': ts,
            'base': [p0, p1]
        }
        return g