Ejemplo n.º 1
0
class PITextureController:
    def _initialize(self):
        self._texture = Named_Texture()
        self.reset()

    def update(self, frame):
        if frame.yuv_buffer is not None:
            self._texture.update_from_yuv_buffer(frame.yuv_buffer, frame.width,
                                                 frame.height)
            self.shape = frame.height, frame.width, 3
        else:
            self._texture.update_from_ndarray(frame.bgr)
            self.shape = frame.bgr.shape

    def draw(self):
        try:
            self._texture.draw()
        except AttributeError:
            self._initialize()
            self._texture.draw()

    def reset(self):
        placeholder = np.ones((1080, 1088, 3), dtype=np.uint8) * 158
        self._texture.update_from_ndarray(placeholder)
        self.shape = placeholder.shape
Ejemplo n.º 2
0
    def generate_heatmap(self, section):

        if self.cache is None:
            logger.warning('Surface cache is not build yet.')
            return

        x, y = self.real_world_size['x'], self.real_world_size['y']
        x = max(1, int(x))
        y = max(1, int(y))

        filter_size = (int(self.heatmap_detail * x) / 2) * 2 + 1
        std_dev = filter_size / 6.
        self.heatmap = np.ones((y, x, 4), dtype=np.uint8)
        all_gaze = []

        for frame_idx, c_e in enumerate(self.cache[section]):
            if c_e:
                frame_idx += section.start
                for gp in self.gaze_on_srf_by_frame_idx(
                        frame_idx, c_e['m_from_screen']):
                    if gp['confidence'] >= self.g_pool.min_data_confidence:
                        all_gaze.append(gp['norm_pos'])

        if not all_gaze:
            logger.warning("No gaze data on surface for heatmap found.")
            all_gaze.append((-1., -1.))
        all_gaze = np.array(all_gaze)
        all_gaze *= [self.real_world_size['x'], self.real_world_size['y']]
        hist, xedge, yedge = np.histogram2d(
            all_gaze[:, 0],
            all_gaze[:, 1],
            bins=[x, y],
            range=[[0, self.real_world_size['x']],
                   [0, self.real_world_size['y']]],
            normed=False,
            weights=None)

        hist = np.rot90(hist)

        #smoothing..
        hist = cv2.GaussianBlur(hist, (filter_size, filter_size), std_dev)
        maxval = np.amax(hist)
        if maxval:
            scale = 255. / maxval
        else:
            scale = 0

        hist = np.uint8(hist * (scale))

        #colormapping
        c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)

        self.heatmap[:, :, :3] = c_map
        self.heatmap[:, :, 3] = 125
        self.heatmap_texture = Named_Texture()
        self.heatmap_texture.update_from_ndarray(self.heatmap)
Ejemplo n.º 3
0
    def __init__(self, g_pool, name="unnamed", saved_definition=None):
        self.g_pool = g_pool
        self.name = name
        self.markers = {}
        self.detected_markers = 0
        self.defined = False
        self.build_up_status = 0
        self.required_build_up = 90.
        self.detected = False
        self.m_to_screen = None
        self.m_from_screen = None
        self.camera_pose_3d = None
        self.use_distortion = True

        self.uid = str(time())
        self.real_world_size = {'x': 1., 'y': 1.}

        self.heatmap = np.ones(0)
        self.heatmap_detail = .2
        self.heatmap_texture = Named_Texture()
        self.gaze_history = deque()
        self.gaze_history_length = 1.0  # unit: seconds

        # window and gui vars
        self._window = None
        self.fullscreen = False
        self.window_should_open = False
        self.window_should_close = False

        self.gaze_on_srf = [
        ]  # points on surface for realtime feedback display
        self.fixations_on_srf = []  # fixations on surface

        self.glfont = fontstash.Context()
        self.glfont.add_font('opensans', get_opensans_font_path())
        self.glfont.set_size(22)
        self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))

        self.old_corners_robust = None
        if saved_definition is not None:
            self.load_from_dict(saved_definition)

        # UI Platform tweaks
        if system() == 'Linux':
            self.window_position_default = (0, 0)
        elif system() == 'Windows':
            self.window_position_default = (8, 90)
        else:
            self.window_position_default = (0, 0)
Ejemplo n.º 4
0
    def generate_heatmap(self,section):

        if self.cache is None:
            logger.warning('Surface cache is not build yet.')
            return


        x,y = self.real_world_size['x'],self.real_world_size['y']
        x = max(1,int(x))
        y = max(1,int(y))

        filter_size = int(int(self.heatmap_detail * x)/2)*2 +1
        std_dev = int(filter_size /6.)
        self.heatmap = np.ones((y,x,4),dtype=np.uint8)
        all_gaze = []

        for frame_idx,c_e in enumerate(self.cache[section]):
            if c_e:
                frame_idx+=section.start
                for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']):
                    if gp['confidence']>=self.g_pool.min_data_confidence:
                        all_gaze.append(gp['norm_pos'])

        if not all_gaze:
            logger.warning("No gaze data on surface for heatmap found.")
            all_gaze.append((-1.,-1.))
        all_gaze = np.array(all_gaze)
        all_gaze *= [self.real_world_size['x'],self.real_world_size['y']]
        hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1],
                                            bins=[x,y],
                                            range=[[0, self.real_world_size['x']], [0,self.real_world_size['y']]],
                                            normed=False,
                                            weights=None)


        hist = np.rot90(hist)

        #smoothing..
        hist = cv2.GaussianBlur(hist,(filter_size,filter_size),std_dev)
        maxval = np.amax(hist)
        if maxval:
            scale = 255./maxval
        else:
            scale = 0

        hist = np.uint8( hist*(scale) )

        #colormapping
        c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)

        self.heatmap[:,:,:3] = c_map
        self.heatmap[:,:,3] = 125
        self.heatmap_texture = Named_Texture()
        self.heatmap_texture.update_from_ndarray(self.heatmap)
Ejemplo n.º 5
0
class Reference_Surface(object):
    """docstring for Reference Surface

    The surface coodinate system is 0-1.
    Origin is the bottom left corner, (1,1) is the top right

    The first scalar in the pos vector is width we call this 'u'.
    The second is height we call this 'v'.
    The surface is thus defined by 4 vertecies:
        Our convention is this order: (0,0),(1,0),(1,1),(0,1)

    The surface is supported by a set of n>=1 Markers:
        Each marker has an id, you can not not have markers with the same id twice.
        Each marker has 4 verts (order is the same as the surface verts)
        Each maker vertex has a uv coord that places it on the surface

    When we find the surface in locate() we use the correspondence
    of uv and screen coords of all 4 verts of all detected markers to get the
    surface to screen homography.

    This allows us to get homographies for partially visible surfaces,
    all we need are 2 visible markers. (We could get away with just
    one marker but in pracise this is to noisy.)
    The more markers we find the more accurate the homography.

    """
    def __init__(self, g_pool, name="unnamed", saved_definition=None):
        self.g_pool = g_pool
        self.name = name
        self.markers = {}
        self.detected_markers = 0
        self.defined = False
        self.build_up_status = 0
        self.required_build_up = 90.
        self.detected = False
        self.m_to_screen = None
        self.m_from_screen = None
        self.camera_pose_3d = None
        self.use_distortion = True

        self.uid = str(time())
        self.real_world_size = {'x': 1., 'y': 1.}

        self.heatmap = np.ones(0)
        self.heatmap_detail = .2
        self.heatmap_texture = Named_Texture()
        self.gaze_history = deque()
        self.gaze_history_length = 1.0  # unit: seconds

        # window and gui vars
        self._window = None
        self.fullscreen = False
        self.window_should_open = False
        self.window_should_close = False

        self.gaze_on_srf = [
        ]  # points on surface for realtime feedback display

        self.glfont = fontstash.Context()
        self.glfont.add_font('opensans', get_opensans_font_path())
        self.glfont.set_size(22)
        self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))

        self.old_corners_robust = None
        if saved_definition is not None:
            self.load_from_dict(saved_definition)

        # UI Platform tweaks
        if system() == 'Linux':
            self.window_position_default = (0, 0)
        elif system() == 'Windows':
            self.window_position_default = (8, 31)
        else:
            self.window_position_default = (0, 0)

    def save_to_dict(self):
        """
        save all markers and name of this surface to a dict.
        """
        markers = dict([(m_id, m.uv_coords.tolist())
                        for m_id, m in self.markers.items()])
        return {
            'name': self.name,
            'uid': self.uid,
            'markers': markers,
            'real_world_size': self.real_world_size,
            'gaze_history_length': self.gaze_history_length
        }

    def load_from_dict(self, d):
        """
        load all markers of this surface to a dict.
        """
        self.name = d['name']
        self.uid = d['uid']
        self.gaze_history_length = d.get('gaze_history_length',
                                         self.gaze_history_length)
        self.real_world_size = d.get('real_world_size', {'x': 1., 'y': 1.})

        marker_dict = d['markers']
        for m_id, uv_coords in marker_dict.items():
            self.markers[m_id] = Support_Marker(m_id)
            self.markers[m_id].load_uv_coords(np.asarray(uv_coords))

        # flag this surface as fully defined
        self.defined = True
        self.build_up_status = self.required_build_up

    def build_correspondence(self, visible_markers, min_marker_perimeter,
                             min_id_confidence):
        """
        - use all visible markers
        - fit a convex quadrangle around it
        - use quadrangle verts to establish perpective transform
        - map all markers into surface space
        - build up list of found markers and their uv coords
        """
        usable_markers = [
            m for m in visible_markers
            if m['perimeter'] >= min_marker_perimeter
        ]
        all_verts = [m['verts'] for m in usable_markers]
        if not all_verts:
            return
        all_verts = np.array(all_verts, dtype=np.float32)
        all_verts.shape = (
            -1, 1, 2)  # [vert,vert,vert,vert,vert...] with vert = [[r,c]]
        # all_verts_undistorted_normalized centered in img center flipped in y and range [-1,1]
        all_verts_undistorted_normalized = self.g_pool.capture.intrinsics.undistortPoints(
            all_verts, use_distortion=self.use_distortion)
        hull = cv2.convexHull(all_verts_undistorted_normalized.astype(
            np.float32),
                              clockwise=False)

        #simplify until we have excatly 4 verts
        if hull.shape[0] > 4:
            new_hull = cv2.approxPolyDP(hull, epsilon=1, closed=True)
            if new_hull.shape[0] >= 4:
                hull = new_hull
        if hull.shape[0] > 4:
            curvature = abs(GetAnglesPolyline(hull, closed=True))
            most_acute_4_threshold = sorted(curvature)[3]
            hull = hull[curvature <= most_acute_4_threshold]

        # all_verts_undistorted_normalized space is flipped in y.
        # we need to change the order of the hull vertecies
        hull = hull[[1, 0, 3, 2], :, :]

        # now we need to roll the hull verts until we have the right orientation:
        # all_verts_undistorted_normalized space has its origin at the image center.
        # adding 1 to the coordinates puts the origin at the top left.
        distance_to_top_left = np.sqrt((hull[:, :, 0] + 1)**2 +
                                       (hull[:, :, 1] + 1)**2)
        bot_left_idx = np.argmin(distance_to_top_left) + 1
        hull = np.roll(hull, -bot_left_idx, axis=0)

        #based on these 4 verts we calculate the transformations into a 0,0 1,1 square space
        m_from_undistored_norm_space = m_verts_from_screen(hull)
        self.detected = True
        # map the markers vertices into the surface space (one can think of these as texture coordinates u,v)
        marker_uv_coords = cv2.perspectiveTransform(
            all_verts_undistorted_normalized, m_from_undistored_norm_space)
        marker_uv_coords.shape = (
            -1, 4, 1, 2)  #[marker,marker...] marker = [ [[r,c]],[[r,c]] ]

        # build up a dict of discovered markers. Each with a history of uv coordinates
        for m, uv in zip(usable_markers, marker_uv_coords):
            try:
                self.markers[m['id']].add_uv_coords(uv)
            except KeyError:
                self.markers[m['id']] = Support_Marker(m['id'])
                self.markers[m['id']].add_uv_coords(uv)

        #average collection of uv correspondences across detected markers
        self.build_up_status = sum(
            [len(m.collected_uv_coords)
             for m in self.markers.values()]) / float(len(self.markers))

        if self.build_up_status >= self.required_build_up:
            self.finalize_correspondence()

    def finalize_correspondence(self):
        """
        - prune markers that have been visible in less than x percent of frames.
        - of those markers select a good subset of uv coords and compute mean.
        - this mean value will be used from now on to estable surface transform
        """
        persistent_markers = {}
        for k, m in self.markers.items():
            if len(m.collected_uv_coords) > self.required_build_up * .5:
                persistent_markers[k] = m
        self.markers = persistent_markers
        for m in self.markers.values():
            m.compute_robust_mean()

        self.defined = True
        if hasattr(self, 'on_finish_define'):
            self.on_finish_define()
            del self.on_finish_define

    def update_gaze_history(self):
        self.gaze_history.extend(self.gaze_on_srf)
        try:  # use newest gaze point to determine age threshold
            age_threshold = self.gaze_history[-1][
                'timestamp'] - self.gaze_history_length
            while self.gaze_history[1]['timestamp'] < age_threshold:
                self.gaze_history.popleft()  # remove outdated gaze points
        except IndexError:
            pass

    def generate_heatmap(self):
        data = [
            gp['norm_pos'] for gp in self.gaze_history
            if gp['confidence'] > 0.6
        ]
        self._generate_heatmap(data)

    def _generate_heatmap(self, data):
        if not data:
            return

        grid = int(self.real_world_size['y']), int(self.real_world_size['x'])

        xvals, yvals = zip(*((x, 1. - y) for x, y in data))
        hist, *edges = np.histogram2d(yvals,
                                      xvals,
                                      bins=grid,
                                      range=[[0, 1.], [0, 1.]],
                                      normed=False)
        filter_h = int(self.heatmap_detail * grid[0]) // 2 * 2 + 1
        filter_w = int(self.heatmap_detail * grid[1]) // 2 * 2 + 1
        hist = cv2.GaussianBlur(hist, (filter_h, filter_w), 0)

        hist_max = hist.max()
        hist *= (255. / hist_max) if hist_max else 0.
        hist = hist.astype(np.uint8)
        c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)
        # reuse allocated memory if possible
        if self.heatmap.shape != (*grid, 4):
            self.heatmap = np.ones((*grid, 4), dtype=np.uint8)
            self.heatmap[:, :, 3] = 125
        self.heatmap[:, :, :3] = c_map
        self.heatmap_texture.update_from_ndarray(self.heatmap)

    def gl_display_heatmap(self):
        if self.detected:
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1, -1, 1)  # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            # apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.heatmap_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()

    def locate(
        self,
        visible_markers,
        min_marker_perimeter,
        min_id_confidence,
        locate_3d=False,
    ):
        """
        - find overlapping set of surface markers and visible_markers
        - compute homography (and inverse) based on this subset
        """

        if not self.defined:
            self.build_correspondence(visible_markers, min_marker_perimeter,
                                      min_id_confidence)

        res = self._get_location(visible_markers, min_marker_perimeter,
                                 min_id_confidence, locate_3d)
        self.detected = res['detected']
        self.detected_markers = res['detected_markers']
        self.m_to_screen = res['m_to_screen']
        self.m_from_screen = res['m_from_screen']
        self.camera_pose_3d = res['camera_pose_3d']

    def _get_location(self,
                      visible_markers,
                      min_marker_perimeter,
                      min_id_confidence,
                      locate_3d=False):

        filtered_markers = [
            m for m in visible_markers
            if m['perimeter'] >= min_marker_perimeter
            and m['id_confidence'] > min_id_confidence
        ]
        marker_by_id = {}
        #if an id shows twice we use the bigger marker (usually this is a screen camera echo artifact.)
        for m in filtered_markers:
            if m["id"] in marker_by_id and m["perimeter"] < marker_by_id[
                    m['id']]['perimeter']:
                pass
            else:
                marker_by_id[m["id"]] = m

        visible_ids = set(marker_by_id.keys())
        requested_ids = set(self.markers.keys())
        overlap = visible_ids & requested_ids
        # need at least two markers per surface when the surface is more that 1 marker.
        if overlap and len(overlap) >= min(2, len(requested_ids)):
            detected = True
            xy = np.array([marker_by_id[i]['verts'] for i in overlap])
            uv = np.array([self.markers[i].uv_coords for i in overlap])
            uv.shape = (-1, 1, 2)

            # our camera lens creates distortions we want to get a good 2d estimate despite that so we:
            # compute the homography transform from marker into the undistored normalized image space
            # (the line below is the same as what you find in methods.undistort_unproject_pts, except that we ommit the z corrd as it is always one.)
            xy_undistorted_normalized = self.g_pool.capture.intrinsics.undistortPoints(
                xy.reshape(-1, 2), use_distortion=self.use_distortion)

            m_to_undistored_norm_space, mask = cv2.findHomography(
                uv,
                xy_undistorted_normalized,
                method=cv2.RANSAC,
                ransacReprojThreshold=100)
            if not mask.all():
                detected = False
            m_from_undistored_norm_space, mask = cv2.findHomography(
                xy_undistorted_normalized, uv)
            # project the corners of the surface to undistored space
            corners_undistored_space = cv2.perspectiveTransform(
                marker_corners_norm.reshape(-1, 1, 2),
                m_to_undistored_norm_space)
            # project and distort these points  and normalize them
            corners_redistorted = self.g_pool.capture.intrinsics.projectPoints(
                cv2.convertPointsToHomogeneous(corners_undistored_space),
                use_distortion=self.use_distortion)
            corners_nulldistorted = self.g_pool.capture.intrinsics.projectPoints(
                cv2.convertPointsToHomogeneous(corners_undistored_space),
                use_distortion=self.use_distortion)

            #normalize to pupil norm space
            corners_redistorted.shape = -1, 2
            corners_redistorted /= self.g_pool.capture.intrinsics.resolution
            corners_redistorted[:, -1] = 1 - corners_redistorted[:, -1]

            #normalize to pupil norm space
            corners_nulldistorted.shape = -1, 2
            corners_nulldistorted /= self.g_pool.capture.intrinsics.resolution
            corners_nulldistorted[:, -1] = 1 - corners_nulldistorted[:, -1]

            # maps for extreme lens distortions will behave irratically beyond the image bounds
            # since our surfaces often extend beyond the screen we need to interpolate
            # between a distored projection and undistored one.

            # def ratio(val):
            #     centered_val = abs(.5 - val)
            #     # signed distance to img cennter .5 is imag bound
            #     # we look to interpolate between .7 and .9
            #     inter = max()

            corners_robust = []
            for nulldist, redist in zip(corners_nulldistorted,
                                        corners_redistorted):
                if -.4 < nulldist[0] < 1.4 and -.4 < nulldist[1] < 1.4:
                    corners_robust.append(redist)
                else:
                    corners_robust.append(nulldist)

            corners_robust = np.array(corners_robust)

            if self.old_corners_robust is not None and np.mean(
                    np.abs(corners_robust - self.old_corners_robust)) < 0.02:
                smooth_corners_robust = self.old_corners_robust
                smooth_corners_robust += .5 * (corners_robust -
                                               self.old_corners_robust)

                corners_robust = smooth_corners_robust
                self.old_corners_robust = smooth_corners_robust
            else:
                self.old_corners_robust = corners_robust

            #compute a perspective thransform from from the marker norm space to the apparent image.
            # The surface corners will be at the right points
            # However the space between the corners may be distored due to distortions of the lens,
            m_to_screen = m_verts_to_screen(corners_robust)
            m_from_screen = m_verts_from_screen(corners_robust)

            camera_pose_3d = None
            if locate_3d:
                # 3d marker support pose estimation:
                # scale normalized object points to world space units (think m,cm,mm)
                uv.shape = -1, 2
                uv *= [self.real_world_size['x'], self.real_world_size['y']]
                # convert object points to lie on z==0 plane in 3d space
                uv3d = np.zeros((uv.shape[0], uv.shape[1] + 1))
                uv3d[:, :-1] = uv
                xy.shape = -1, 1, 2
                # compute pose of object relative to camera center
                is3dPoseAvailable, rot3d_cam_to_object, translate3d_cam_to_object = self.g_pool.capture.intrinsics.solvePnP(
                    uv3d, xy)
                print("{} \t {} \t {}".format(translate3d_cam_to_object[0],
                                              translate3d_cam_to_object[1],
                                              translate3d_cam_to_object[2]))

                if is3dPoseAvailable:

                    # not verifed, potentially usefull info: http://stackoverflow.com/questions/17423302/opencv-solvepnp-tvec-units-and-axes-directions

                    ###marker posed estimation from virtually projected points.
                    # object_pts = np.array([[[0,0],[0,1],[1,1],[1,0]]],dtype=np.float32)
                    # projected_pts = cv2.perspectiveTransform(object_pts,self.m_to_screen)
                    # projected_pts.shape = -1,2
                    # projected_pts *= img_size
                    # projected_pts.shape = -1, 1, 2
                    # # scale object points to world space units (think m,cm,mm)
                    # object_pts.shape = -1,2
                    # object_pts *= self.real_world_size
                    # # convert object points to lie on z==0 plane in 3d space
                    # object_pts_3d = np.zeros((4,3))
                    # object_pts_3d[:,:-1] = object_pts
                    # self.is3dPoseAvailable, rot3d_cam_to_object, translate3d_cam_to_object = cv2.solvePnP(object_pts_3d, projected_pts, K, dist_coef,flags=cv2.CV_EPNP)

                    # transformation from Camera Optical Center:
                    #   first: translate from Camera center to object origin.
                    #   second: rotate x,y,z
                    #   coordinate system is x,y,z where z goes out from the camera into the viewed volume.
                    # print rot3d_cam_to_object[0],rot3d_cam_to_object[1],rot3d_cam_to_object[2], translate3d_cam_to_object[0],translate3d_cam_to_object[1],translate3d_cam_to_object[2]

                    #turn translation vectors into 3x3 rot mat.
                    rot3d_cam_to_object_mat, _ = cv2.Rodrigues(
                        rot3d_cam_to_object)

                    #to get the transformation from object to camera we need to reverse rotation and translation
                    translate3d_object_to_cam = -translate3d_cam_to_object
                    # rotation matrix inverse == transpose
                    rot3d_object_to_cam_mat = rot3d_cam_to_object_mat.T

                    # we assume that the volume of the object grows out of the marker surface and not into it. We thus have to flip the z-Axis:
                    flip_z_axix_hm = np.eye(4, dtype=np.float32)
                    flip_z_axix_hm[2, 2] = -1
                    # create a homogenous tranformation matrix from the rotation mat
                    rot3d_object_to_cam_hm = np.eye(4, dtype=np.float32)
                    rot3d_object_to_cam_hm[:-1, :-1] = rot3d_object_to_cam_mat
                    # create a homogenous tranformation matrix from the translation vect
                    translate3d_object_to_cam_hm = np.eye(4, dtype=np.float32)
                    translate3d_object_to_cam_hm[:-1,
                                                 -1] = translate3d_object_to_cam.reshape(
                                                     3)

                    # combine all tranformations into transformation matrix that decribes the move from object origin and orientation to camera origin and orientation
                    tranform3d_object_to_cam = np.matrix(
                        flip_z_axix_hm) * np.matrix(
                            rot3d_object_to_cam_hm) * np.matrix(
                                translate3d_object_to_cam_hm)
                    camera_pose_3d = tranform3d_object_to_cam
            if detected == False:
                camera_pose_3d = None
                m_from_screen = None
                m_to_screen = None
                m_from_undistored_norm_space = None
                m_to_undistored_norm_space = None

        else:
            detected = False
            camera_pose_3d = None
            m_from_screen = None
            m_to_screen = None
            m_from_undistored_norm_space = None
            m_to_undistored_norm_space = None

        return {
            'detected': detected,
            'detected_markers': len(overlap),
            'm_from_undistored_norm_space': m_from_undistored_norm_space,
            'm_to_undistored_norm_space': m_to_undistored_norm_space,
            'm_from_screen': m_from_screen,
            'm_to_screen': m_to_screen,
            'camera_pose_3d': camera_pose_3d
        }

    def img_to_ref_surface(self, pos):
        #convenience lines to allow 'simple' vectors (x,y) to be used
        shape = pos.shape
        pos.shape = (-1, 1, 2)
        new_pos = cv2.perspectiveTransform(pos, self.m_from_screen)
        new_pos.shape = shape
        return new_pos

    def ref_surface_to_img(self, pos):
        #convenience lines to allow 'simple' vectors (x,y) to be used
        shape = pos.shape
        pos.shape = (-1, 1, 2)
        new_pos = cv2.perspectiveTransform(pos, self.m_to_screen)
        new_pos.shape = shape
        return new_pos

    @staticmethod
    def map_datum_to_surface(d, m_from_screen):
        pos = np.array([d['norm_pos']]).reshape(1, 1, 2)
        mapped_pos = cv2.perspectiveTransform(pos, m_from_screen)
        mapped_pos.shape = (2)
        on_srf = bool((0 <= mapped_pos[0] <= 1) and (0 <= mapped_pos[1] <= 1))
        return {
            'topic': d['topic'] + "_on_surface",
            'norm_pos': (mapped_pos[0], mapped_pos[1]),
            'confidence': d['confidence'],
            'on_srf': on_srf,
            'base_data': d,
            'timestamp': d['timestamp']
        }

    def map_data_to_surface(self, data, m_from_screen):
        return [self.map_datum_to_surface(d, m_from_screen) for d in data]

    def move_vertex(self, vert_idx, new_pos):
        """
        this fn is used to manipulate the surface boundary (coordinate system)
        new_pos is in uv-space coords
        if we move one vertex of the surface we need to find
        the tranformation from old quadrangle to new quardangle
        and apply that transformation to our marker uv-coords
        """
        before = marker_corners_norm
        after = before.copy()
        after[vert_idx] = new_pos
        transform = cv2.getPerspectiveTransform(after, before)
        for m in self.markers.values():
            m.uv_coords = cv2.perspectiveTransform(m.uv_coords, transform)

    def add_marker(self, marker, visible_markers, min_marker_perimeter,
                   min_id_confidence):
        '''
        add marker to surface.
        '''
        res = self._get_location(visible_markers,
                                 min_marker_perimeter,
                                 min_id_confidence,
                                 locate_3d=False)
        if res['detected']:
            support_marker = Support_Marker(marker['id'])
            marker_verts = np.array(marker['verts'])
            marker_verts.shape = (-1, 1, 2)
            if self.use_distortion:
                marker_verts_undistorted_normalized = self.g_pool.capture.intrinsics.undistortPoints(
                    marker_verts)
            else:
                marker_verts_undistorted_normalized = self.g_pool.capture.intrinsics.undistortPoints(
                    marker_verts)
            marker_uv_coords = cv2.perspectiveTransform(
                marker_verts_undistorted_normalized,
                res['m_from_undistored_norm_space'])
            support_marker.load_uv_coords(marker_uv_coords)
            self.markers[marker['id']] = support_marker

    def remove_marker(self, marker):
        if len(self.markers) == 1:
            logger.warning(
                "Need at least one marker per surface. Will not remove this last marker."
            )
            return
        self.markers.pop(marker['id'])

    def marker_status(self):
        return "{}   {}/{}".format(self.name, self.detected_markers,
                                   len(self.markers))

    def get_mode_toggle(self, pos, img_shape):
        if self.detected and self.defined:
            x, y = pos
            frame = np.array([[[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]],
                             dtype=np.float32)
            frame = cv2.perspectiveTransform(frame, self.m_to_screen)
            text_anchor = frame.reshape((5, -1))[2]
            text_anchor[1] = 1 - text_anchor[1]
            text_anchor *= img_shape[1], img_shape[0]
            text_anchor = text_anchor[0], text_anchor[1] - 75
            surface_edit_anchor = text_anchor[0], text_anchor[1] + 25
            marker_edit_anchor = text_anchor[0], text_anchor[1] + 50
            if np.sqrt((x - surface_edit_anchor[0])**2 +
                       (y - surface_edit_anchor[1])**2) < 15:
                return 'surface_mode'
            elif np.sqrt((x - marker_edit_anchor[0])**2 +
                         (y - marker_edit_anchor[1])**2) < 15:
                return 'marker_mode'
            else:
                return None
        else:
            return None

    def gl_draw_frame(self,
                      img_size,
                      color=(1.0, 0.2, 0.6, 1.0),
                      highlight=False,
                      surface_mode=False,
                      marker_mode=False):
        """
        draw surface and markers
        """
        if self.detected:
            r, g, b, a = color
            frame = np.array([[[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]],
                             dtype=np.float32)
            hat = np.array([[[.3, .7], [.7, .7], [.5, .9], [.3, .7]]],
                           dtype=np.float32)
            hat = cv2.perspectiveTransform(hat, self.m_to_screen)
            frame = cv2.perspectiveTransform(frame, self.m_to_screen)
            alpha = min(1, self.build_up_status / self.required_build_up)
            if highlight:
                draw_polyline_norm(frame.reshape((5, 2)),
                                   1,
                                   RGBA(r, g, b, a * .1),
                                   line_type=GL_POLYGON)
            draw_polyline_norm(frame.reshape((5, 2)), 1,
                               RGBA(r, g, b, a * alpha))
            draw_polyline_norm(hat.reshape((4, 2)), 1,
                               RGBA(r, g, b, a * alpha))
            text_anchor = frame.reshape((5, -1))[2]
            text_anchor[1] = 1 - text_anchor[1]
            text_anchor *= img_size[1], img_size[0]
            text_anchor = text_anchor[0], text_anchor[1] - 75
            surface_edit_anchor = text_anchor[0], text_anchor[1] + 25
            marker_edit_anchor = text_anchor[0], text_anchor[1] + 50
            if self.defined:
                if marker_mode:
                    draw_points([marker_edit_anchor], color=RGBA(0, .8, .7))
                else:
                    draw_points([marker_edit_anchor])
                if surface_mode:
                    draw_points([surface_edit_anchor], color=RGBA(0, .8, .7))
                else:
                    draw_points([surface_edit_anchor])

                self.glfont.set_blur(3.9)
                self.glfont.set_color_float((0, 0, 0, .8))
                self.glfont.draw_text(text_anchor[0] + 15, text_anchor[1] + 6,
                                      self.marker_status())
                self.glfont.draw_text(surface_edit_anchor[0] + 15,
                                      surface_edit_anchor[1] + 6,
                                      'edit surface')
                self.glfont.draw_text(marker_edit_anchor[0] + 15,
                                      marker_edit_anchor[1] + 6,
                                      'add/remove markers')
                self.glfont.set_blur(0.0)
                self.glfont.set_color_float((0.1, 8., 8., .9))
                self.glfont.draw_text(text_anchor[0] + 15, text_anchor[1] + 6,
                                      self.marker_status())
                self.glfont.draw_text(surface_edit_anchor[0] + 15,
                                      surface_edit_anchor[1] + 6,
                                      'edit surface')
                self.glfont.draw_text(marker_edit_anchor[0] + 15,
                                      marker_edit_anchor[1] + 6,
                                      'add/remove markers')
            else:
                progress = (self.build_up_status /
                            float(self.required_build_up)) * 100
                progress_text = '%.0f%%' % progress
                self.glfont.set_blur(3.9)
                self.glfont.set_color_float((0, 0, 0, .8))
                self.glfont.draw_text(text_anchor[0] + 15, text_anchor[1] + 6,
                                      self.marker_status())
                self.glfont.draw_text(surface_edit_anchor[0] + 15,
                                      surface_edit_anchor[1] + 6,
                                      'Learning affiliated markers...')
                self.glfont.draw_text(marker_edit_anchor[0] + 15,
                                      marker_edit_anchor[1] + 6, progress_text)
                self.glfont.set_blur(0.0)
                self.glfont.set_color_float((0.1, 8., 8., .9))
                self.glfont.draw_text(text_anchor[0] + 15, text_anchor[1] + 6,
                                      self.marker_status())
                self.glfont.draw_text(surface_edit_anchor[0] + 15,
                                      surface_edit_anchor[1] + 6,
                                      'Learning affiliated markers...')
                self.glfont.draw_text(marker_edit_anchor[0] + 15,
                                      marker_edit_anchor[1] + 6, progress_text)

    def gl_draw_corners(self):
        """
        draw surface and markers
        """
        if self.detected:
            frame = cv2.perspectiveTransform(
                marker_corners_norm.reshape(-1, 1, 2), self.m_to_screen)
            draw_points_norm(frame.reshape((4, 2)), 20,
                             RGBA(1.0, 0.2, 0.6, .5))

    #### fns to draw surface in seperate window
    def gl_display_in_window(self, world_tex):
        """
        here we map a selected surface onto a seperate window.
        """
        if self._window and self.detected:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            clear_gl_screen()

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_from_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1, -1, 1)  # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            world_tex.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()

            # now lets get recent pupil positions on this surface:
            for gp in self.gaze_on_srf:
                draw_points_norm([gp['norm_pos']],
                                 color=RGBA(0.0, 0.8, 0.5, 0.8),
                                 size=80)

            glfwSwapBuffers(self._window)
            glfwMakeContextCurrent(active_window)

    #### fns to draw surface in separate window
    def gl_display_in_window_3d(self, world_tex):
        """
        here we map a selected surface onto a seperate window.
        """
        K, img_size = self.g_pool.capture.intrinsics.K, self.g_pool.capture.intrinsics.resolution

        if self._window and self.camera_pose_3d is not None:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            glClearColor(.8, .8, .8, 1.)

            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
            glClearDepth(1.0)
            glDepthFunc(GL_LESS)
            glEnable(GL_DEPTH_TEST)
            self.trackball.push()

            glMatrixMode(GL_MODELVIEW)

            draw_coordinate_system(l=self.real_world_size['x'])
            glPushMatrix()
            glScalef(self.real_world_size['x'], self.real_world_size['y'], 1)
            draw_polyline([[0, 0], [0, 1], [1, 1], [1, 0]],
                          color=RGBA(.5, .3, .1, .5),
                          thickness=3)
            glPopMatrix()
            # Draw the world window as projected onto the plane using the homography mapping
            glPushMatrix()
            glScalef(self.real_world_size['x'], self.real_world_size['y'], 1)
            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_from_screen)
            glMultMatrixf(m)
            glTranslatef(0, 0, -.01)
            world_tex.draw()
            draw_polyline([[0, 0], [0, 1], [1, 1], [1, 0]],
                          color=RGBA(.5, .3, .6, .5),
                          thickness=3)
            glPopMatrix()

            # Draw the camera frustum and origin using the 3d tranformation obtained from solvepnp
            glPushMatrix()
            glMultMatrixf(self.camera_pose_3d.T.flatten())
            draw_frustum(img_size, K, 150)
            glLineWidth(1)
            draw_frustum(img_size, K, .1)
            draw_coordinate_system(l=5)
            glPopMatrix()

            self.trackball.pop()

            glfwSwapBuffers(self._window)
            glfwMakeContextCurrent(active_window)

    def open_window(self):
        if not self._window:
            if self.fullscreen:
                monitor = glfwGetMonitors()[self.monitor_idx]
                mode = glfwGetVideoMode(monitor)
                height, width = mode[0], mode[1]
            else:
                monitor = None
                height, width = 640, int(
                    640. /
                    (self.real_world_size['x'] / self.real_world_size['y'])
                )  #open with same aspect ratio as surface

            self._window = glfwCreateWindow(height,
                                            width,
                                            "Reference Surface: " + self.name,
                                            monitor=monitor,
                                            share=glfwGetCurrentContext())
            if not self.fullscreen:
                glfwSetWindowPos(self._window, self.window_position_default[0],
                                 self.window_position_default[1])

            self.trackball = Trackball()
            self.input = {'down': False, 'mouse': (0, 0)}

            #Register callbacks
            glfwSetFramebufferSizeCallback(self._window, self.on_resize)
            glfwSetKeyCallback(self._window, self.on_window_key)
            glfwSetWindowCloseCallback(self._window, self.on_close)
            glfwSetMouseButtonCallback(self._window,
                                       self.on_window_mouse_button)
            glfwSetCursorPosCallback(self._window, self.on_pos)
            glfwSetScrollCallback(self._window, self.on_scroll)

            self.on_resize(self._window, *glfwGetFramebufferSize(self._window))

            # gl_state settings
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            basic_gl_setup()
            make_coord_system_norm_based()

            # refresh speed settings
            glfwSwapInterval(0)

            glfwMakeContextCurrent(active_window)

    def close_window(self):
        if self._window:
            glfwDestroyWindow(self._window)
            self._window = None
            self.window_should_close = False

    def open_close_window(self):
        if self._window:
            self.close_window()
        else:
            self.open_window()

    # window calbacks
    def on_resize(self, window, w, h):
        self.trackball.set_window_size(w, h)
        active_window = glfwGetCurrentContext()
        glfwMakeContextCurrent(window)
        adjust_gl_view(w, h)
        glfwMakeContextCurrent(active_window)

    def on_window_key(self, window, key, scancode, action, mods):
        if action == GLFW_PRESS:
            if key == GLFW_KEY_ESCAPE:
                self.on_close()

    def on_close(self, window=None):
        self.close_window()

    def on_window_mouse_button(self, window, button, action, mods):
        if action == GLFW_PRESS:
            self.input['down'] = True
            self.input['mouse'] = glfwGetCursorPos(window)
        if action == GLFW_RELEASE:
            self.input['down'] = False

    def on_pos(self, window, x, y):
        if self.input['down']:
            old_x, old_y = self.input['mouse']
            self.trackball.drag_to(x - old_x, y - old_y)
            self.input['mouse'] = x, y

    def on_scroll(self, window, x, y):
        self.trackball.zoom_to(y)

    def cleanup(self):
        if self._window:
            self.close_window()
Ejemplo n.º 6
0
def world(pupil_queue, timebase, lauchner_pipe, eye_pipes, eyes_are_alive,
          user_dir, version, cap_src):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    import logging
    # Set up root logger for this process before doing imports of logged modules.
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(user_dir, 'world.log'), mode='w')
    fh.setLevel(logger.level)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level + 10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        'World Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter(
        'WORLD Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    #silence noisy modules
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logging.getLogger("libav").setLevel(logging.ERROR)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmfull but unnessasary.

    #general imports
    from time import time
    import numpy as np

    #display
    import glfw
    from pyglui import ui, graph, cygl
    from pyglui.cygl.utils import Named_Texture
    from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based

    #check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version
    assert pyglui_version >= '0.8'

    #monitoring
    import psutil

    # helpers/utils
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t
    from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
    from version_utils import VersionFormat
    import audio

    # Plug-ins
    from plugin import Plugin_List, import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins
    from recorder import Recorder
    from show_calibration import Show_Calibration
    from display_recent_gaze import Display_Recent_Gaze
    from pupil_server import Pupil_Server
    from pupil_sync import Pupil_Sync
    from marker_detector import Marker_Detector
    from log_display import Log_Display
    from annotations import Annotation_Capture
    # create logger for the context of this function

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (0, 0)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (8, 31)
    else:
        scroll_factor = 1.0
        window_position_default = (0, 0)

    #g_pool holds variables for this process
    g_pool = Global_Container()

    # make some constants avaiable
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.app = 'capture'
    g_pool.pupil_queue = pupil_queue
    g_pool.timebase = timebase
    # g_pool.lauchner_pipe = lauchner_pipe
    g_pool.eye_pipes = eye_pipes
    g_pool.eyes_are_alive = eyes_are_alive

    #manage plugins
    runtime_plugins = import_runtime_plugins(
        os.path.join(g_pool.user_dir, 'plugins'))
    user_launchable_plugins = [
        Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector,
        Annotation_Capture
    ] + runtime_plugins
    system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
    plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index, plugin_by_index))
    default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
                       ('Display_Recent_Gaze', {}),
                       ('Screen_Marker_Calibration', {}), ('Recorder', {})]

    # Callback functions
    def on_resize(window, w, h):
        if not g_pool.iconified:
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfw.glfwGetCursorPos(window)
        pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
        pos = denormalize(
            pos,
            (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfw.glfwGetFramebufferSize(window)[0] /
            glfw.glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    tick = delta_t()

    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_world'))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()

    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size': (1280, 720), 'frame_rate': 30}
    previous_settings = session_settings.get('capture_settings', None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        lauchner_pipe.send("Exit")
        return

    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get(
        'pupil_confidence_threshold', .6)
    g_pool.detection_mapping_mode = session_settings.get(
        'detection_mapping_mode', '2d')
    g_pool.active_calibration_plugin = None

    audio.audio_mode = session_settings.get('audio_mode',
                                            audio.default_audio_mode)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def launch_eye_process(eye_id, blocking=False):
        if eyes_are_alive[eye_id].value:
            logger.error("Eye%s process already running." % eye_id)
            return
        lauchner_pipe.send(eye_id)
        eye_pipes[eye_id].send(
            ('Set_Detection_Mapping_Mode', g_pool.detection_mapping_mode))

        if blocking:
            #wait for ready message from eye to sequentialize startup
            eye_pipes[eye_id].send('Ping')
            eye_pipes[eye_id].recv()

        logger.warning('Eye %s process started.' % eye_id)

    def stop_eye_process(eye_id, blocking=False):
        if eyes_are_alive[eye_id].value:
            eye_pipes[eye_id].send('Exit')
            if blocking:
                raise NotImplementedError()

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        if new_mode == '2d':
            for p in g_pool.plugins:
                if "Vector_Gaze_Mapper" in p.class_name:
                    logger.warning(
                        "The gaze mapper is not supported in 2d mode. Please recalibrate."
                    )
                    p.alive = False
            g_pool.plugins.clean()
        for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes):
            if alive.value:
                pipe.send(('Set_Detection_Mapping_Mode', new_mode))
        g_pool.detection_mapping_mode = new_mode

    #window and gl setup
    glfw.glfwInit()
    width, height = session_settings.get('window_size',
                                         (frame.width, frame.height))
    main_window = glfw.glfwCreateWindow(width, height, "World")
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-350, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface size'))
    general_settings.append(
        ui.Button(
            'Reset window size', lambda: glfw.glfwSetWindowSize(
                main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('audio_mode', audio, selection=audio.audio_modes))
    general_settings.append(
        ui.Selector('detection_mapping_mode',
                    g_pool,
                    label='detection & mapping mode',
                    setter=set_detection_mapping_mode,
                    selection=['2d', '3d']))
    general_settings.append(
        ui.Switch('eye0_process',
                  label='Detect eye 0',
                  setter=lambda alive: start_stop_eye(0, alive),
                  getter=lambda: eyes_are_alive[0].value))
    general_settings.append(
        ui.Switch('eye1_process',
                  label='Detect eye 1',
                  setter=lambda alive: start_stop_eye(1, alive),
                  getter=lambda: eyes_are_alive[1].value))
    general_settings.append(
        ui.Selector('Open plugin',
                    selection=user_launchable_plugins,
                    labels=[
                        p.__name__.replace('_', ' ')
                        for p in user_launchable_plugins
                    ],
                    setter=open_plugin,
                    getter=lambda: "Select to load"))
    general_settings.append(
        ui.Slider('pupil_confidence_threshold',
                  g_pool,
                  step=.01,
                  min=0.,
                  max=1.,
                  label='Minimum pupil confidence'))
    general_settings.append(
        ui.Info_Text('Capture Version: %s' % g_pool.version))
    g_pool.sidebar.append(general_settings)

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.capture.init_gui(g_pool.sidebar)

    #plugins that are loaded based on user settings from previous session
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(
        g_pool, plugin_by_name,
        session_settings.get('loaded_plugins', default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(
        0,
        ui.Selector(
            'active_calibration_plugin',
            getter=lambda: g_pool.active_calibration_plugin.__class__,
            selection=calibration_plugins,
            labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
            setter=open_plugin,
            label='Method'))

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfw.glfwSetKeyCallback(main_window, on_key)
    glfw.glfwSetCharCallback(main_window, on_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    # refresh speed settings
    glfw.glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    if session_settings.get('eye1_process_alive', False):
        launch_eye_process(1, blocking=True)
    if session_settings.get('eye0_process_alive', True):
        launch_eye_process(0, blocking=False)

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        #a dictionary that allows plugins to post and read events
        events = {}

        #report time between now and the last loop interation
        events['dt'] = get_dt()

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p['confidence'])
        events['pupil_positions'] = recent_pupil_positions

        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifications:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        if g_pool.iconified:
            pass
        else:
            g_pool.image_tex.update_from_frame(frame)

        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height, frame.width, 3))
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        if not g_pool.iconified:
            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings[
        'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings['eye0_process_alive'] = eyes_are_alive[0].value
    session_settings['eye1_process_alive'] = eyes_are_alive[1].value
    session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
    session_settings['audio_mode'] = audio.audio_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()
    cap.close()

    #shut down eye processes:
    stop_eye_process(0)
    stop_eye_process(1)

    #shut down laucher
    lauchner_pipe.send("Exit")

    logger.debug("world process done")
Ejemplo n.º 7
0
def session(rec_dir):
    plugin_dir = os.path.join(user_dir, 'plugins')
    if not os.path.isdir(plugin_dir):
        os.mkdir(plugin_dir)
    runtime_plugins = import_runtime_plugins(plugin_dir)

    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
                          Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__)

    analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
                               Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
                               Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__)

    other_plugins = sorted([Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__)
    user_plugins = sorted(runtime_plugins, key=lambda x: x.__name__)

    user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos, glfwGetWindowSize(window))
        pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    update_recording_to_recent(rec_dir)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)
    app_version = get_version(version_file)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    timestamps = np.load(timestamps_path)

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'

    # Initialize capture
    cap = File_Source(g_pool, video_path, timestamps=list(timestamps))

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if VersionFormat(session_settings.get("version", '0.0')) < get_version(version_file):
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfwGetFramebufferSize(main_window))

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']
    g_pool.pupil_data = pupil_data
    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = timestamps
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index()-2)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.delayed_notifications[notification['subject']] = notification

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark)
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(export_range.start, export_range.stop))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.notifications.append(notification)

    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
    g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    selector_label = "Select to load"

    vis_labels = ["   " + p.__name__.replace('_', ' ') for p in vis_plugins]
    analysis_labels = ["   " + p.__name__.replace('_', ' ') for p in analysis_plugins]
    other_labels = ["   " + p.__name__.replace('_', ' ') for p in other_plugins]
    user_labels = ["   " + p.__name__.replace('_', ' ') for p in user_plugins]

    plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label]
               + other_plugins + [selector_label] + user_plugins)
    labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"]
              + other_labels + ["User added"] + user_labels)

    g_pool.main_menu.append(ui.Selector('Open plugin:',
                                        selection=plugins,
                                        labels=labels,
                                        setter=open_plugin,
                                        getter=lambda: selector_label))

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)
    glfwSetDropCallback(main_window, on_drop)

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"
    g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]

    # trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    while not glfwWindowShouldClose(main_window):
        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        events['frame'] = frame
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
               ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # publish delayed notifiactions when their time has come.
        for n in list(g_pool.delayed_notifications.values()):
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        g_pool.gui.update()

        # present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
Ejemplo n.º 8
0
Archivo: eye.py Proyecto: sleip87/pupil
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url,
        user_dir, version, eye_id, overwrite_cap_settings=None):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools
    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):

        # logging setup
        import logging
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logger = logging.getLogger()
        logger.handlers = []
        logger.setLevel(logging.INFO)
        logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        # general imports
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible
        from ui_roi import UIRoi
        # monitoring
        import psutil

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, AV_Writer
        from ndsi import H264Writer
        from video_capture import source_classes
        from video_capture import manager_classes

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {Detector_2D.__name__: Detector_2D,
                           Detector_3D.__name__: Detector_3D}

        # UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 10.0
            window_position_default = (600,31+ 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        # g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.process = 'eye{}'.format(eye_id)
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value
        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window, w, h):
            if is_window_visible(window):
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
                g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
                g_pool.gui.update_window(w, h)
                g_pool.gui.collect_menus()
                for g in g_pool.graphs:
                    g.scale = hdpi_factor
                    g.adjust_window_size(w, h)
                adjust_gl_view(w, h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            hdpi_factor = glfw.glfwGetFramebufferSize(
                window)[0] / glfw.glfwGetWindowSize(window)[0]
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,g_pool.capture.frame_size )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode('utf-8') for x in range(count)]
            g_pool.capture_manager.on_drop(paths)
            g_pool.capture.on_drop(paths)

        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id)))
        if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
            logger.info("Session setting are from a different version of this app. I will not use those.")
            session_settings.clear()


        g_pool.iconified = False
        g_pool.capture = None
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get('flip', False)
        g_pool.display_mode = session_settings.get(
            'display_mode', 'camera_image')
        g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                         'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
                                         'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}


        capture_manager_settings = session_settings.get(
            'capture_manager_settings', ('UVC_Manager',{}))

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__:c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)


        if eye_id == 0:
            cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
        else:
            cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]

        # Initialize capture
        default_settings = ('UVC_Source',{
                            'preferred_names'  : cap_src,
                            'frame_size': (640,480),
                            'frame_rate': 90
                            })

        capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
        source_class_name, source_settings = capture_source_settings
        source_class_by_name = {c.__name__:c for c in source_classes}
        g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
        assert g_pool.capture

        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get('roi')
        if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        pupil_detector_settings = session_settings.get(
            'pupil_detector_settings', None)
        last_pupil_detector = pupil_detectors[session_settings.get(
            'last_pupil_detector', Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(
            g_pool, pupil_detector_settings)

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)

        # Initialize glfw
        glfw.glfwInit()
        title = "Pupil Capture - eye {}".format(eye_id)
        width, height = session_settings.get(
            'window_size', g_pool.capture.frame_size)
        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get(
            'window_position', window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                           pos=(-300, 0),
                                           size=(0, 0),
                                           header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(ui.Selector('gui_user_scale', g_pool,
                                          setter=set_scale,
                                          selection=[.8, .9, 1., 1.1, 1.2],
                                          label='Interface Size'))
        general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) )
        general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
        general_settings.append(ui.Selector('display_mode',
                                            g_pool,
                                            setter=set_display_mode_info,
                                            selection=['camera_image','roi','algorithm'],
                                            labels=['Camera Image', 'ROI', 'Algorithm'],
                                            label="Mode")
                                            )
        g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])

        general_settings.append(g_pool.display_mode_info)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector('pupil_detector',
                                        getter=lambda: g_pool.pupil_detector.__class__,
                                        setter=set_detector, selection=[
                                            Detector_2D, Detector_3D],
                                        labels=['C++ 2d detector',
                                                'C++ 3d detector'],
                                        label="Detection method")
        general_settings.append(detector_selector)

        g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
        g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
        g_pool.capture_source_menu.collapsed = True
        g_pool.capture.init_gui()

        g_pool.sidebar.append(general_settings)
        g_pool.sidebar.append(g_pool.capture_selector_menu)
        g_pool.sidebar.append(g_pool.capture_source_menu)

        g_pool.pupil_detector.init_gui(g_pool.sidebar)

        g_pool.capture_manager.init_gui()
        g_pool.writer = None

        def replace_source(source_class_name,source_settings):
            g_pool.capture.cleanup()
            g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
            g_pool.capture.init_gui()
            if g_pool.writer:
                logger.info("Done recording.")
                g_pool.writer.release()
                g_pool.writer = None

        g_pool.replace_source = replace_source # for ndsi capture

        def replace_manager(manager_class):
            g_pool.capture_manager.cleanup()
            g_pool.capture_manager = manager_class(g_pool)
            g_pool.capture_manager.init_gui()

        #We add the capture selection menu, after a manager has been added:
        g_pool.capture_selector_menu.insert(0,ui.Selector(
                                                'capture_manager',g_pool,
                                                setter    = replace_manager,
                                                getter    = lambda: g_pool.capture_manager.__class__,
                                                selection = manager_classes,
                                                labels    = [b.gui_name for b in manager_classes],
                                                label     = 'Manager'
                                            ))


        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = 'jpeg'

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning('Process started.')

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification['subject']
                if subject.startswith('eye_process.should_stop'):
                    if notification['eye_id'] == eye_id:
                        break
                elif subject == 'set_detection_mapping_mode':
                    if notification['mode'] == '3d':
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    else:
                        if not isinstance(g_pool.pupil_detector, Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                elif subject == 'recording.started':
                    if notification['record_eye'] and g_pool.capture.online:
                        record_path = notification['rec_path']
                        raw_mode = notification['compression']
                        logger.info("Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id))
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate)
                        elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'):
                            g_pool.writer = H264Writer(video_path,
                                                       g_pool.capture.frame_size[0],
                                                       g_pool.capture.frame_size[1],
                                                       g_pool.capture.frame_rate)
                        else:
                            g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate)
                elif subject == 'recording.stopped':
                    if g_pool.writer:
                        logger.info("Done recording.")
                        g_pool.writer.release()
                        g_pool.writer = None
                elif subject.startswith('meta.should_doc'):
                    ipc_socket.notify({
                        'subject': 'meta.doc',
                        'actor': 'eye{}'.format(eye_id),
                        'doc': eye.__doc__
                    })
                elif subject.startswith('frame_publishing.started'):
                    should_publish_frames = True
                    frame_publish_format = notification.get('format', 'jpeg')
                elif subject.startswith('frame_publishing.stopped'):
                    should_publish_frames = False
                    frame_publish_format = 'jpeg'
                elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process:
                    replace_source(notification['name'],notification['args'])

                g_pool.capture.on_notify(notification)

            # Get an image from the grabber
            event = {}
            g_pool.capture.recent_events(event)
            frame = event.get('frame')
            g_pool.capture_manager.recent_events(event)
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width):
                    g_pool.pupil_detector.on_resolution_change((g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size)
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        else:
                            raise AttributeError()
                    except AttributeError:
                        pass
                    else:
                        pupil_socket.send('frame.eye.%s'%eye_id,{
                            'width': frame.width,
                            'height': frame.height,
                            'index': frame.index,
                            'timestamp': frame.timestamp,
                            'format': frame_publish_format,
                            '__raw_data__': [data]
                        })

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1./dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                # pupil ellipse detection
                result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
                result['id'] = eye_id

                # stream the result
                pupil_socket.send('pupil.%s'%eye_id,result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    if frame:
                        # switch to work in normalized coordinate space
                        if g_pool.display_mode == 'algorithm':
                            g_pool.image_tex.update_from_ndarray(frame.img)
                        elif g_pool.display_mode in ('camera_image', 'roi'):
                            g_pool.image_tex.update_from_ndarray(frame.gray)
                        else:
                            pass
                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()
                    f_width, f_height = g_pool.capture.frame_size
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    if frame:
                        if result['method'] == '3d c++':
                            eye_ball = result['projected_sphere']
                            try:
                                pts = cv2.ellipse2Poly(
                                    (int(eye_ball['center'][0]),
                                     int(eye_ball['center'][1])),
                                    (int(eye_ball['axes'][0] / 2),
                                     int(eye_ball['axes'][1] / 2)),
                                    int(eye_ball['angle']), 0, 360, 8)
                            except ValueError as e:
                                pass
                            else:
                                draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence']))
                        if result['confidence'] > 0:
                            if 'ellipse' in result:
                                pts = cv2.ellipse2Poly(
                                    (int(result['ellipse']['center'][0]),
                                     int(result['ellipse']['center'][1])),
                                    (int(result['ellipse']['axes'][0] / 2),
                                     int(result['ellipse']['axes'][1] / 2)),
                                    int(result['ellipse']['angle']), 0, 360, 15)
                                confidence = result['confidence'] * 0.7
                                draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
                                draw_points([result['ellipse']['center']],
                                            size=20,
                                            color=RGBA(1., 0., 0., confidence),
                                            sharpness=1.)

                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    g_pool.gui.update()

                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize()  # detector decides if we visualize or not

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer = None

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict()
        session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.capture.deinit_gui()
        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()
        logger.info("Process shutting down.")
Ejemplo n.º 9
0
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version,
        eye_id, cap_src):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """
    is_alive = Is_Alive_Manager(is_alive_flag)
    with is_alive:
        import logging
        # Set up root logger for this process before doing imports of logged modules.
        logger = logging.getLogger()
        logger.setLevel(logging.INFO)
        # remove inherited handlers
        logger.handlers = []
        # create file handler which logs even debug messages
        fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id),
                                 mode='w')
        # fh.setLevel(logging.DEBUG)
        # create console handler with a higher log level
        ch = logging.StreamHandler()
        ch.setLevel(logger.level + 10)
        # create formatter and add it to the handlers
        formatter = logging.Formatter(
            'Eye' + str(eye_id) +
            ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        formatter = logging.Formatter(
            'EYE' + str(eye_id) +
            ' Process [%(levelname)s] %(name)s : %(message)s')
        ch.setFormatter(formatter)
        # add the handlers to the logger
        logger.addHandler(fh)
        logger.addHandler(ch)
        #silence noisy modules
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logging.getLogger("libav").setLevel(logging.ERROR)
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        # We deferr the imports becasue of multiprocessing.
        # Otherwise the world process each process also loads the other imports.

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture
        from OpenGL.GL import GL_LINE_LOOP
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based
        from ui_roi import UIRoi
        #monitoring
        import psutil

        # helpers/utils
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
        from av_writer import JPEG_Writer, AV_Writer

        # Pupil detectors
        from pupil_detectors import Canny_Detector, Detector_2D, Detector_3D
        pupil_detectors = {
            Canny_Detector.__name__: Canny_Detector,
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D
        }

        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600, 31 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.pupil_queue = pupil_queue
        g_pool.timebase = timebase

        # Callback functions
        def on_resize(window, w, h):
            if not g_pool.iconified:
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w, h)
                graph.adjust_size(w, h)
                adjust_gl_view(w, h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_button(window, button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return  # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    pos = denormalize(
                        pos,
                        (frame.width, frame.height))  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                            pos, g_pool.u_r.handle_size + 40,
                            g_pool.u_r.handle_size + 40):
                        return  # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, (frame.width, frame.height))
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
        if session_settings.get("version",
                                VersionFormat('0.0')) < g_pool.version:
            logger.info(
                "Session setting are from older version of this app. I will not use those."
            )
            session_settings.clear()
        # Initialize capture
        cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
        default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
        previous_settings = session_settings.get('capture_settings', None)
        if previous_settings and previous_settings['name'] == cap.name:
            cap.settings = previous_settings
        else:
            cap.settings = default_settings

        # Test capture
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Could not retrieve image from capture")
            cap.close()
            return

        #signal world that we are ready to go
        # pipe_to_world.send('eye%s process ready'%eye_id)

        # any object we attach to the g_pool object *from now on* will only be visible to this process!
        # vars should be declared here to make them visible to the code reader.
        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.flip = session_settings.get('flip', False)
        g_pool.display_mode = session_settings.get('display_mode',
                                                   'camera_image')
        g_pool.display_mode_info_text = {
            'camera_image':
            "Raw eye camera image. This uses the least amount of CPU power",
            'roi':
            "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
            'algorithm':
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
        }

        g_pool.u_r = UIRoi(frame.img.shape)
        g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))

        def on_frame_size_change(new_size):
            g_pool.u_r = UIRoi((new_size[1], new_size[0]))

        cap.on_frame_size_change = on_frame_size_change

        writer = None

        pupil_detector_settings = session_settings.get(
            'pupil_detector_settings', None)
        last_pupil_detector = pupil_detectors[session_settings.get(
            'last_pupil_detector', Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,
                                                    pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)

        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s" % eye_id
        width, height = session_settings.get('window_size',
                                             (frame.width, frame.height))
        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_frame(frame)
        glfw.glfwSwapInterval(0)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale', 1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                           pos=(-300, 0),
                                           size=(0, 0),
                                           header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(
            ui.Slider('scale',
                      g_pool.gui,
                      setter=set_scale,
                      step=.05,
                      min=1.,
                      max=2.5,
                      label='Interface Size'))
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, frame.width, frame.height)))
        general_settings.append(
            ui.Switch('flip', g_pool, label='Flip image display'))
        general_settings.append(
            ui.Selector('display_mode',
                        g_pool,
                        setter=set_display_mode_info,
                        selection=['camera_image', 'roi', 'algorithm'],
                        labels=['Camera Image', 'ROI', 'Algorithm'],
                        label="Mode"))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.sidebar.append(general_settings)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector(
            'pupil_detector',
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Canny_Detector, Detector_2D, Detector_3D],
            labels=[
                'Python 2D detector', 'C++ 2d detector', 'C++ 3d detector'
            ],
            label="Detection method")
        general_settings.append(detector_selector)

        # let detector add its GUI
        g_pool.pupil_detector.init_gui(g_pool.sidebar)
        # let the camera add its GUI
        g_pool.capture.init_gui(g_pool.sidebar)

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_key)
        glfw.glfwSetCharCallback(main_window, on_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = frame.timestamp

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"

        #create a timer to control window update frequency
        window_update_timer = timer(1 / 60.)

        def window_should_update():
            return next(window_update_timer)

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if pipe_to_world.poll():
                cmd = pipe_to_world.recv()
                if cmd == 'Exit':
                    break
                elif cmd == "Ping":
                    pipe_to_world.send("Pong")
                    command = None
                else:
                    command, payload = cmd
                if command == 'Set_Detection_Mapping_Mode':
                    if payload == '3d':
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    else:
                        set_detector(Detector_2D)
                        detector_selector.read_only = False

            else:
                command = None

            # Get an image from the grabber
            try:
                frame = cap.get_frame()
            except CameraCaptureError:
                logger.error("Capture from Camera Failed. Stopping.")
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                cap.seek_to_frame(0)
                frame = cap.get_frame()

            #update performace graphs
            t = frame.timestamp
            dt, ts = t - ts, t
            try:
                fps_graph.add(1. / dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()

            ###  RECORDING of Eye Video (on demand) ###
            # Setup variables and lists for recording
            if 'Rec_Start' == command:
                record_path, raw_mode = payload
                logger.info("Will save eye video to: %s" % record_path)
                timestamps_path = os.path.join(record_path,
                                               "eye%s_timestamps.npy" % eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = JPEG_Writer(video_path, cap.frame_rate)
                else:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = AV_Writer(video_path, cap.frame_rate)
                timestamps = []
            elif 'Rec_Stop' == command:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)

            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(
                frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            g_pool.pupil_queue.put(result)

            # GL drawing
            if window_should_update():
                if not g_pool.iconified:
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image', 'roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()
                    # switch to work in pixel space
                    make_coord_system_pixel_based(
                        (frame.height, frame.width, 3), g_pool.flip)

                    if result['confidence'] > 0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly(
                                (int(result['ellipse']['center'][0]),
                                 int(result['ellipse']['center'][1])),
                                (int(result['ellipse']['axes'][0] / 2),
                                 int(result['ellipse']['axes'][1] / 2)),
                                int(result['ellipse']['angle']), 0, 360, 15)
                            draw_polyline(pts, 1, RGBA(1., 0, 0, .5))
                        draw_points([result['ellipse']['center']],
                                    size=20,
                                    color=RGBA(1., 0., 0., .5),
                                    sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize(
                )  #detector decides if we visualize or not

        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path, np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = g_pool.version
        session_settings[
            'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        cap.close()

        logger.debug("Process done")
Ejemplo n.º 10
0
 def _initialize(self):
     self._texture = Named_Texture()
     self.reset()
Ejemplo n.º 11
0
def world(g_pool,cap_src,cap_size):
    """world
    Creates a window, gl context.
    Grabs images from a capture.
    Receives Pupil coordinates from eye process[es]
    Can run various plug-ins.
    """

    #manage plugins
    runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
    user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins
    system_plugins  = [Log_Display,Display_Recent_Gaze,Recorder]
    plugin_by_index =  system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index,plugin_by_index))
    default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]



    # Callback functions
    def on_resize(window,w, h):
        if not g_pool.iconified:
            g_pool.gui.update_window(w,h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w,h)
            adjust_gl_view(w,h)
            for p in g_pool.plugins:
                p.on_window_resize(window,w,h)

    def on_iconify(window,iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key,scancode,action,mods)

    def on_char(window,char):
        g_pool.gui.update_char(char)

    def on_button(window,button, action, mods):
        g_pool.gui.update_button(button,action,mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos,glfwGetWindowSize(main_window))
        pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos,button,action)

    def on_pos(window,x, y):
        hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
        x,y = x*hdpi_factor,y*hdpi_factor
        g_pool.gui.update_mouse(x,y)

    def on_scroll(window,x,y):
        g_pool.gui.update_scroll(x,y*scroll_factor)


    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')


    tick = delta_t()
    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
    if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
        logger.info("Session setting are from older version of this app. I will not use those.")
        session_settings.clear()

    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size':cap_size,'frame_rate':24}
    previous_settings = session_settings.get('capture_settings',None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return


    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.iconified = False

    g_pool.capture = cap
    g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
    g_pool.active_calibration_plugin = None


    def open_plugin(plugin):
        if plugin ==  "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()


    #window and gl setup
    glfwInit()
    width,height = session_settings.get('window_size',(frame.width, frame.height))
    main_window = glfwCreateWindow(width,height, "World")
    window_pos = session_settings.get('window_position',window_position_default)
    glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale',1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
    general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
    general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
                                        setter= open_plugin, getter=lambda: "Select to load"))
    general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
    general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
    g_pool.sidebar.append(general_settings)

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.sidebar.append(g_pool.calibration_menu)
    g_pool.gui.append(g_pool.sidebar)
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
    g_pool.gui.append(g_pool.quickbar)
    g_pool.capture.init_gui(g_pool.sidebar)

    #plugins that are loaded based on user settings from previous session
    g_pool.notifications = []
    g_pool.delayed_notifications = {}
    g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
                                        labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
                                        setter= open_plugin,label='Method'))

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window,on_resize)
    glfwSetWindowCloseCallback(main_window,on_close)
    glfwSetWindowIconifyCallback(main_window,on_iconify)
    glfwSetKeyCallback(main_window,on_key)
    glfwSetCharCallback(main_window,on_char)
    glfwSetMouseButtonCallback(main_window,on_button)
    glfwSetCursorPosCallback(main_window,on_pos)
    glfwSetScrollCallback(main_window,on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)

    # refresh speed settings
    glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config',{})



    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20,130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140,130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260,130)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    # Event loop
    while not g_pool.quit.value:

        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from camera failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video file is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt,ts = t-ts,t
        try:
            fps_graph.add(1./dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()


        #a dictionary that allows plugins to post and read events
        events = {}

        #report time between now and the last loop interation
        events['dt'] = get_dt()

        #receive and map pupil positions
        recent_pupil_positions = []
        while not g_pool.pupil_queue.empty():
            p = g_pool.pupil_queue.get()
            recent_pupil_positions.append(p)
            pupil_graph.add(p['confidence'])
        events['pupil_positions'] = recent_pupil_positions


        # publish delayed notifiactions when their time has come.
        for n in g_pool.delayed_notifications.values():
            if n['_notify_time_'] < time():
                del n['_notify_time_']
                del g_pool.delayed_notifications[n['subject']]
                g_pool.notifications.append(n)

        # notify each plugin if there are new notifications:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame,events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        if g_pool.iconified:
            pass
        else:
            g_pool.image_tex.update_from_frame(frame)

        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height,frame.width,3))
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        if not g_pool.iconified:
            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfwSwapBuffers(main_window)
        glfwPollEvents()

    glfwRestoreWindow(main_window) #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    logger.debug("Process done")
Ejemplo n.º 12
0
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
    hide_ui=False,
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
        ``set_detection_mapping_mode``: Sets detection method
        ``eye_process.should_stop``: Stops the eye process
        ``recording.started``: Starts recording eye video
        ``recording.stopped``: Stops recording eye video
        ``frame_publishing.started``: Starts frame publishing
        ``frame_publishing.stopped``: Stops frame publishing
        ``start_eye_plugin``: Start plugins in eye process

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport
        from ui_roi import UIRoi

        # monitoring
        import psutil

        # Plug-ins
        from plugin import Plugin_List

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, MPEG_Writer
        from ndsi import H264Writer
        from video_capture import source_classes, manager_classes

        from background_helper import IPC_Logging_Task_Proxy
        from pupil_detector_plugins import available_detector_plugins
        from pupil_detector_plugins.manager import PupilDetectorManager

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            # NOTE: Interrupt is handled in world/service/player which are responsible for
            # shutting down the eye process properly

        signal.signal(signal.SIGINT, interrupt_handler)

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = "capture"
        g_pool.eye_id = eye_id
        g_pool.process = f"eye{eye_id}"
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        default_detector_cls, available_detectors = available_detector_plugins(
        )
        plugins = (manager_classes + source_classes + available_detectors +
                   [PupilDetectorManager])
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        preferred_names = [
            f"Pupil Cam3 ID{eye_id}",
            f"Pupil Cam2 ID{eye_id}",
            f"Pupil Cam1 ID{eye_id}",
        ]
        if eye_id == 0:
            preferred_names += ["HD-6000"]
        default_capture_settings = (
            "UVC_Source",
            {
                "preferred_names": preferred_names,
                "frame_size": (320, 240),
                "frame_rate": 120,
            },
        )

        default_plugins = [
            # TODO: extend with plugins
            default_capture_settings,
            ("UVC_Manager", {}),
            # Detector needs to be loaded first to set `g_pool.pupil_detector`
            (default_detector_cls.__name__, {}),
            ("PupilDetectorManager", {}),
        ]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor

            active_window = glfw.glfwGetCurrentContext()
            glfw.glfwMakeContextCurrent(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x *= hdpi_factor
            y *= hdpi_factor
            g_pool.gui.update_mouse(x, y)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, g_pool.capture.frame_size)
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir,
                         "user_settings_eye{}".format(eye_id)))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.iconified = False
        g_pool.capture = None
        g_pool.flip = session_settings.get("flip", False)
        g_pool.display_mode = session_settings.get("display_mode",
                                                   "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image":
            "Raw eye camera image. This uses the least amount of CPU power",
            "roi":
            "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm":
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.glfwInit()
        if hide_ui:
            glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0)  # hide window
        title = "Pupil Capture - eye {}".format(eye_id)

        width, height = session_settings.get("window_size",
                                             (640 + icon_bar_width, 480))

        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get("window_position",
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(
            np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2],
                label="Interface Size",
            ))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width *= 2
            f_height *= 2
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        def uroi_on_mouse_button(button, action, mods):
            if g_pool.display_mode == "roi":
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    x *= hdpi_factor
                    y *= hdpi_factor
                    pos = normalize((x, y), camera_render_size)
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(
                        pos,
                        g_pool.capture.frame_size)  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,
                                                     g_pool.u_r.handle_size,
                                                     g_pool.u_r.handle_size):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            ))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])

        general_settings.append(g_pool.display_mode_info)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)
        toggle_general_settings(False)

        plugins_to_load = session_settings.get("loaded_plugins",
                                               default_plugins)
        if overwrite_cap_settings:
            # Ensure that overwrite_cap_settings takes preference over source plugins
            # with incorrect settings that were loaded from session settings.
            plugins_to_load.append(overwrite_cap_settings)

        g_pool.plugins = Plugin_List(g_pool, plugins_to_load)

        g_pool.writer = None

        g_pool.u_r = UIRoi(
            (g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get("roi")
        if roi_user_settings and tuple(
                roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        record_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        start_time_synced = notification["start_time_synced"]
                        logger.info(
                            "Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(record_path,
                                                  "eye{}.mp4".format(eye_id))
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(
                                video_path, start_time_synced)
                        elif hasattr(g_pool.capture._recent_frame,
                                     "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = MPEG_Writer(
                                video_path, start_time_synced)
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify({
                        "subject": "meta.doc",
                        "actor": "eye{}".format(eye_id),
                        "doc": eye.__doc__,
                    })
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (subject.startswith("start_eye_plugin")
                      and notification["target"] == g_pool.process):
                    try:
                        g_pool.plugins.add(
                            g_pool.plugin_by_name[notification["name"]],
                            notification.get("args", {}),
                        )
                    except KeyError as err:
                        logger.error(f"Attempt to load unknown plugin: {err}")

                for plugin in g_pool.plugins:
                    plugin.on_notify(notification)

            event = {}
            for plugin in g_pool.plugins:
                plugin.recent_events(event)

            frame = event.get("frame")
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
                        f_height,
                        f_width,
                ):
                    g_pool.pupil_detector.on_resolution_change(
                        (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
                        g_pool.capture.frame_size,
                    )
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.
                                format(type(frame), frame_publish_format))
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send({
                            "topic":
                            "frame.eye.{}".format(eye_id),
                            "width":
                            frame.width,
                            "height":
                            frame.height,
                            "index":
                            frame.index,
                            "timestamp":
                            frame.timestamp,
                            "format":
                            frame_publish_format,
                            "__raw_data__": [data],
                        })

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                result = event.get("pupil_detection_result", None)
                if result is not None:
                    pupil_socket.send(result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    glViewport(0, 0, *camera_render_size)
                    for p in g_pool.plugins:
                        p.gl_display()

                    glViewport(0, 0, *camera_render_size)
                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == "roi":
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    glViewport(0, 0, *window_size)
                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    try:
                        clipboard = glfw.glfwGetClipboardString(
                            main_window).decode()
                    except AttributeError:  # clipboard is None, might happen on startup
                        clipboard = ""
                    g_pool.gui.update_clipboard(clipboard)
                    user_input = g_pool.gui.update()
                    if user_input.clipboard != clipboard:
                        # only write to clipboard if content changed
                        glfw.glfwSetClipboardString(
                            main_window, user_input.clipboard.encode())

                    for button, action, mods in user_input.buttons:
                        x, y = glfw.glfwGetCursorPos(main_window)
                        pos = x * hdpi_factor, y * hdpi_factor
                        pos = normalize(pos, camera_render_size)
                        # Position in img pixels
                        pos = denormalize(pos, g_pool.capture.frame_size)

                        for plugin in g_pool.plugins:
                            if plugin.on_click(pos, button, action):
                                break

                    for key, scancode, action, mods in user_input.keys:
                        for plugin in g_pool.plugins:
                            if plugin.on_key(key, scancode, action, mods):
                                break

                    for char_ in user_input.chars:
                        for plugin in g_pool.plugins:
                            if plugin.on_char(char_):
                                break

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer.release()
            g_pool.writer = None

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        # save session persistent settings
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["roi"] = g_pool.u_r.get()
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)

        if not hide_ui:
            glfw.glfwRestoreWindow(
                main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.glfwGetWindowPos(
                main_window)
            session_window_size = glfw.glfwGetWindowSize(main_window)
            if 0 not in session_window_size:
                session_settings["window_size"] = session_window_size

        session_settings.close()

        for plugin in g_pool.plugins:
            plugin.alive = False
        g_pool.plugins.clean()

        glfw.glfwDestroyWindow(main_window)
        g_pool.gui.terminate()
        glfw.glfwTerminate()
        logger.info("Process shutting down.")
Ejemplo n.º 13
0
def world(
    timebase,
    eye_procs_alive,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    preferred_remote_port,
    hide_ui,
    debug,
):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``eye_process.started``
        ``start_plugin``
        ``should_stop``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            "subject": "eye_process.should_start.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": delay,
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            "subject": "eye_process.should_stop.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": 0.2,
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def detection_enabled_getter() -> bool:
        return g_pool.pupil_detection_enabled

    def detection_enabled_setter(is_on: bool):
        g_pool.pupil_detection_enabled = is_on
        n = {"subject": "set_pupil_detection_enabled", "value": is_on}
        ipc_pub.notify(n)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        from OpenGL.GL import GL_COLOR_BUFFER_BIT

        # display
        import glfw

        glfw.ERROR_REPORTING = "raise"

        from version_utils import parse_version
        from pyglui import ui, cygl, __version__ as pyglui_version

        assert parse_version(pyglui_version) >= parse_version(
            "1.27"
        ), "pyglui out of date, please upgrade to newest version"
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic

        logger.info("Application Version: {}".format(version))
        logger.info("System Info: {}".format(get_system_info()))
        logger.debug(f"Debug flag: {debug}")

        import audio

        # Plug-ins
        from plugin import (
            Plugin,
            System_Plugin_Base,
            Plugin_List,
            import_runtime_plugins,
        )
        from plugin_manager import Plugin_Manager
        from calibration_choreography import (
            available_calibration_choreography_plugins,
            CalibrationChoreographyPlugin,
            patch_loaded_plugins_with_choreography_plugin,
        )

        available_choreography_plugins = available_calibration_choreography_plugins()

        from gaze_mapping import registered_gazer_classes
        from gaze_mapping.gazer_base import GazerBase
        from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from network_api import NetworkApiPlugin
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker_Online
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from blink_detection import Blink_Detection
        from video_capture import (
            source_classes,
            manager_classes,
            Base_Manager,
            Base_Source,
        )
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from accuracy_visualizer import Accuracy_Visualizer

        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay
        from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (8, 90)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        content_scale = 1.0

        # g_pool holds variables for this process they are accessible to all plugins
        g_pool = SimpleNamespace()
        g_pool.debug = debug
        g_pool.app = "capture"
        g_pool.process = "world"
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eye_procs_alive = eye_procs_alive
        g_pool.preferred_remote_port = preferred_remote_port

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, "plugins")
        )
        runtime_plugins = [
            p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin)
        ]
        user_plugins = [
            Pupil_Groups,
            NetworkApiPlugin,
            Time_Sync,
            Surface_Tracker_Online,
            Annotation_Capture,
            Log_History,
            Fixation_Detector,
            Blink_Detection,
            Remote_Recorder,
            Accuracy_Visualizer,
            Camera_Intrinsics_Estimation,
            Hololens_Relay,
            Online_Head_Pose_Tracker,
        ]

        system_plugins = (
            [
                Log_Display,
                Display_Recent_Gaze,
                Recorder,
                Pupil_Data_Relay,
                Plugin_Manager,
                System_Graphs,
            ]
            + manager_classes
            + source_classes
        )
        plugins = (
            system_plugins
            + user_plugins
            + runtime_plugins
            + available_choreography_plugins
            + registered_gazer_classes()
        )
        user_plugins += [
            p
            for p in runtime_plugins
            if not isinstance(
                p,
                (
                    Base_Manager,
                    Base_Source,
                    System_Plugin_Base,
                    CalibrationChoreographyPlugin,
                    GazerBase,
                ),
            )
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_name = "UVC_Source"
        default_capture_settings = {
            "preferred_names": [
                "Pupil Cam1 ID2",
                "Logitech Camera",
                "(046d:081d)",
                "C510",
                "B525",
                "C525",
                "C615",
                "C920",
                "C930e",
            ],
            "frame_size": (1280, 720),
            "frame_rate": 30,
        }

        default_plugins = [
            (default_capture_name, default_capture_settings),
            ("Pupil_Data_Relay", {}),
            ("UVC_Manager", {}),
            ("NDSI_Manager", {}),
            ("HMD_Streaming_Manager", {}),
            ("File_Manager", {}),
            ("Log_Display", {}),
            ("Dummy_Gaze_Mapper", {}),
            ("Display_Recent_Gaze", {}),
            # Calibration choreography plugin is added below by calling
            # patch_loaded_plugins_with_choreography_plugin
            ("Recorder", {}),
            ("NetworkApiPlugin", {}),
            ("Fixation_Detector", {}),
            ("Blink_Detection", {}),
            ("Accuracy_Visualizer", {}),
            ("Plugin_Manager", {}),
            ("System_Graphs", {}),
        ]

        def consume_events_and_render_buffer():
            gl_utils.glViewport(0, 0, *camera_render_size)
            for p in g_pool.plugins:
                p.gl_display()

            gl_utils.glViewport(0, 0, *window_size)
            try:
                clipboard = glfw.get_clipboard_string(main_window).decode()
            except (AttributeError, glfw.GLFWError):
                # clipboard is None, might happen on startup
                clipboard = ""
            g_pool.gui.update_clipboard(clipboard)
            user_input = g_pool.gui.update()
            if user_input.clipboard != clipboard:
                # only write to clipboard if content changed
                glfw.set_clipboard_string(main_window, user_input.clipboard)

            for button, action, mods in user_input.buttons:
                x, y = glfw.get_cursor_pos(main_window)
                pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                    main_window, x, y, cached_scale=None
                )
                pos = normalize(pos, camera_render_size)
                # Position in img pixels
                pos = denormalize(pos, g_pool.capture.frame_size)

                for plugin in g_pool.plugins:
                    if plugin.on_click(pos, button, action):
                        break

            for key, scancode, action, mods in user_input.keys:
                for plugin in g_pool.plugins:
                    if plugin.on_key(key, scancode, action, mods):
                        break

            for char_ in user_input.chars:
                for plugin in g_pool.plugins:
                    if plugin.on_char(char_):
                        break

            glfw.swap_buffers(main_window)

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal content_scale
            if w == 0 or h == 0:
                return

            # Always clear buffers on resize to make sure that there are no overlapping
            # artifacts from previous frames.
            gl_utils.glClear(GL_COLOR_BUFFER_BIT)
            gl_utils.glClearColor(0, 0, 0, 1)

            content_scale = gl_utils.get_content_scale(window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(window)
            g_pool.gui.scale = content_scale
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()

            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

            # Minimum window size required, otherwise parts of the UI can cause openGL
            # issues with permanent effects. Depends on the content scale, which can
            # potentially be dynamically modified, so we re-adjust the size limits every
            # time here.
            min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale)
            glfw.set_window_size_limits(
                window,
                min_size,
                min_size,
                glfw.DONT_CARE,
                glfw.DONT_CARE,
            )

            # Needed, to update the window buffer while resizing
            consume_events_and_render_buffer()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
                window, x, y, cached_scale=None
            )
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, paths):
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_world")
        )
        if parse_version(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.min_data_confidence = 0.6
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", 0.8
        )
        g_pool.pupil_detection_enabled = session_settings.get(
            "pupil_detection_enabled", True
        )
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.set_audio_mode(
            session_settings.get("audio_mode", audio.get_default_audio_mode())
        )

        def handle_notifications(noti):
            subject = noti["subject"]
            if subject == "set_pupil_detection_enabled":
                g_pool.pupil_detection_enabled = noti["value"]
            elif subject == "start_plugin":
                try:
                    g_pool.plugins.add(
                        g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})
                    )
                except KeyError as err:
                    logger.error(f"Attempt to load unknown plugin: {err}")
            elif subject == "stop_plugin":
                for p in g_pool.plugins:
                    if p.class_name == noti["name"]:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == "eye_process.started":
                noti = {
                    "subject": "set_pupil_detection_enabled",
                    "value": g_pool.pupil_detection_enabled,
                }
                ipc_pub.notify(noti)
            elif subject == "set_min_calibration_confidence":
                g_pool.min_calibration_confidence = noti["value"]
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify(
                    {"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__}
                )
                for p in g_pool.plugins:
                    if (
                        p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify
                    ):
                        ipc_pub.notify(
                            {
                                "subject": "meta.doc",
                                "actor": p.class_name,
                                "doc": p.on_notify.__doc__,
                            }
                        )
            elif subject == "world_process.adapt_window_size":
                set_window_size()
            elif subject == "world_process.should_stop":
                glfw.set_window_should_close(main_window, True)

        width, height = session_settings.get(
            "window_size", (1280 + icon_bar_width, 720)
        )

        # window and gl setup
        glfw.init()
        glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
        if hide_ui:
            glfw.window_hint(glfw.VISIBLE, 0)  # hide window
        main_window = glfw.create_window(
            width, height, "Pupil Capture - World", None, None
        )

        window_position_manager = gl_utils.WindowPositionManager()
        window_pos = window_position_manager.new_window_position(
            window=main_window,
            default_position=window_position_default,
            previous_position=session_settings.get("window_position", None),
        )
        glfw.set_window_pos(main_window, window_pos[0], window_pos[1])

        glfw.make_context_current(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.set_window_should_close(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0})

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is opened, the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")

        def set_window_size():
            # Get current capture frame size
            f_width, f_height = g_pool.capture.frame_size

            # Get current display scale factor
            content_scale = gl_utils.get_content_scale(main_window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
            display_scale_factor = content_scale / framebuffer_scale

            # Scale the capture frame size by display scale factor
            f_width *= display_scale_factor
            f_height *= display_scale_factor

            # Increas the width to account for the added scaled icon bar width
            f_width += icon_bar_width * display_scale_factor

            # Set the newly calculated size (scaled capture frame size + scaled icon bar width)
            glfw.set_window_size(main_window, int(f_width), int(f_height))

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(
            ui.Selector(
                "Audio mode",
                None,
                getter=audio.get_audio_mode,
                setter=audio.set_audio_mode,
                selection=audio.get_audio_mode_list(),
            )
        )

        general_settings.append(
            ui.Switch(
                "pupil_detection_enabled",
                label="Pupil detection",
                getter=detection_enabled_getter,
                setter=detection_enabled_setter,
            )
        )
        general_settings.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: start_stop_eye(0, alive),
                getter=lambda: eye_procs_alive[0].value,
            )
        )
        general_settings.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: start_stop_eye(1, alive),
                getter=lambda: eye_procs_alive[1].value,
            )
        )

        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(g_pool.version))
        )
        general_settings.append(
            ui.Button("Restart with default settings", reset_restart)
        )

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        loaded_plugins = session_settings.get("loaded_plugins", default_plugins)

        # Resolve the active calibration choreography plugin
        loaded_plugins = patch_loaded_plugins_with_choreography_plugin(
            loaded_plugins, app=g_pool.app
        )
        session_settings["loaded_plugins"] = loaded_plugins

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(g_pool, loaded_plugins)

        if not g_pool.capture:
            # Make sure we always have a capture running. Important if there was no
            # capture stored in session settings.
            g_pool.plugins.add(
                g_pool.plugin_by_name[default_capture_name], default_capture_settings
            )

        # Register callbacks main_window
        glfw.set_framebuffer_size_callback(main_window, on_resize)
        glfw.set_key_callback(main_window, on_window_key)
        glfw.set_char_callback(main_window, on_window_char)
        glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
        glfw.set_cursor_pos_callback(main_window, on_pos)
        glfw.set_scroll_callback(main_window, on_scroll)
        glfw.set_drop_callback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(True)

        # now that we have a proper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})
        # If previously selected plugin was not loaded this time, we will have an
        # expanded menubar without any menu selected. We need to ensure the menubar is
        # collapsed in this case.
        if all(submenu.collapsed for submenu in g_pool.menubar.elements):
            g_pool.menubar.collapsed = True

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.get_framebuffer_size(main_window))

        if session_settings.get("eye1_process_alive", True):
            launch_eye_process(1, delay=0.6)
        if session_settings.get("eye0_process_alive", True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({"subject": "world_process.started"})
        logger.warning("Process started.")

        if platform.system() == "Darwin":
            # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of
            # occluded windows, based on the swap interval value. This causes an FPS drop
            # and leads to problems when recording. To side-step this behaviour, the swap
            # interval is set to zero.
            #
            # Read more about window occlusion on macOS here:
            # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html
            glfw.swap_interval(0)

        # Event loop
        while not glfw.window_should_close(main_window) and not process_was_interrupted:

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # "blacklisted" events that were already sent
            del events["pupil"]
            del events["gaze"]
            # delete if exists. More expensive than del, so only use it when key might not exist
            events.pop("annotation", None)

            # send new events to ipc:
            if "frame" in events:
                del events["frame"]  # send explicitly with frame publisher
            if "depth_frame" in events:
                del events["depth_frame"]
            if "audio_packets" in events:
                del events["audio_packets"]
            del events["dt"]  # no need to send this
            for data in events.values():
                assert isinstance(data, (list, tuple))
                for d in data:
                    ipc_pub.send(d)

            glfw.make_context_current(main_window)
            # render visual feedback from loaded plugins
            glfw.poll_events()
            if window_should_update() and gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                try:
                    clipboard = glfw.get_clipboard_string(main_window).decode()
                except (AttributeError, glfw.GLFWError):
                    # clipboard is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.set_clipboard_string(main_window, user_input.clipboard)

                for button, action, mods in user_input.buttons:
                    x, y = glfw.get_cursor_pos(main_window)
                    pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                        main_window, x, y, cached_scale=None
                    )
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                glfw.swap_buffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)
        session_settings["eye0_process_alive"] = eye_procs_alive[0].value
        session_settings["eye1_process_alive"] = eye_procs_alive[1].value
        session_settings[
            "min_calibration_confidence"
        ] = g_pool.min_calibration_confidence
        session_settings["pupil_detection_enabled"] = g_pool.pupil_detection_enabled
        session_settings["audio_mode"] = audio.get_audio_mode()

        if not hide_ui:
            glfw.restore_window(main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.get_window_pos(main_window)
            session_window_size = glfw.get_window_size(main_window)
            if 0 not in session_window_size:
                f_width, f_height = session_window_size
                if platform.system() in ("Windows", "Linux"):
                    f_width, f_height = (
                        f_width / content_scale,
                        f_height / content_scale,
                    )
                session_settings["window_size"] = int(f_width), int(f_height)

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.destroy_window(main_window)
        glfw.terminate()

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Capture crashed with trace:\n{}".format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "world_process.stopped"})
        sleep(1.0)
Ejemplo n.º 14
0
Archivo: eye.py Proyecto: N-M-T/pupil
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
    hide_ui=False,
    debug=False,
    pub_socket_hwm=None,
    parent_application="capture",
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
        ``eye_process.should_stop``: Stops the eye process
        ``recording.started``: Starts recording eye video
        ``recording.stopped``: Stops recording eye video
        ``frame_publishing.started``: Starts frame publishing
        ``frame_publishing.stopped``: Stops frame publishing
        ``start_eye_plugin``: Start plugins in eye process

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url, pub_socket_hwm)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        from OpenGL.GL import GL_COLOR_BUFFER_BIT

        # display
        import glfw
        from gl_utils import GLFWErrorReporting

        GLFWErrorReporting.set_default()

        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import Named_Texture
        import gl_utils
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport

        # monitoring
        import psutil

        # Plug-ins
        from plugin import Plugin_List

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import parse_version
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError
        from ndsi import H264Writer
        from video_capture import source_classes, manager_classes
        from roi import Roi

        from background_helper import IPC_Logging_Task_Proxy
        from pupil_detector_plugins import available_detector_plugins, EVENT_KEY

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            # NOTE: Interrupt is handled in world/service/player which are responsible for
            # shutting down the eye process properly

        signal.signal(signal.SIGINT, interrupt_handler)

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        content_scale = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.debug = debug
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = parent_application
        g_pool.eye_id = eye_id
        g_pool.process = f"eye{eye_id}"
        g_pool.timebase = timebase
        g_pool.camera_render_size = None

        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_socket
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        def load_runtime_pupil_detection_plugins():
            from plugin import import_runtime_plugins
            from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin

            plugins_path = os.path.join(g_pool.user_dir, "plugins")

            for plugin in import_runtime_plugins(plugins_path):
                if not isinstance(plugin, type):
                    continue
                if not issubclass(plugin, PupilDetectorPlugin):
                    continue
                if plugin is PupilDetectorPlugin:
                    continue
                yield plugin

        available_detectors = available_detector_plugins()
        runtime_detectors = list(load_runtime_pupil_detection_plugins())
        plugins = (
            manager_classes
            + source_classes
            + available_detectors
            + runtime_detectors
            + [Roi]
        )
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        preferred_names = [
            f"Pupil Cam3 ID{eye_id}",
            f"Pupil Cam2 ID{eye_id}",
            f"Pupil Cam1 ID{eye_id}",
        ]
        if eye_id == 0:
            preferred_names += ["HD-6000"]

        default_capture_name = "UVC_Source"
        default_capture_settings = {
            "preferred_names": preferred_names,
            "frame_size": (192, 192),
            "frame_rate": 120,
        }

        default_plugins = [
            # TODO: extend with plugins
            (default_capture_name, default_capture_settings),
            ("UVC_Manager", {}),
            *[(p.__name__, {}) for p in available_detectors],
            ("NDSI_Manager", {}),
            ("HMD_Streaming_Manager", {}),
            ("File_Manager", {}),
            ("Roi", {}),
        ]

        def consume_events_and_render_buffer():
            glfw.make_context_current(main_window)
            clear_gl_screen()

            if all(c > 0 for c in g_pool.camera_render_size):
                glViewport(0, 0, *g_pool.camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

            glViewport(0, 0, *window_size)
            # render graphs
            fps_graph.draw()
            cpu_graph.draw()

            # render GUI
            try:
                clipboard = glfw.get_clipboard_string(main_window).decode()
            except (AttributeError, glfw.GLFWError):
                # clipboard is None, might happen on startup
                clipboard = ""
            g_pool.gui.update_clipboard(clipboard)
            user_input = g_pool.gui.update()
            if user_input.clipboard != clipboard:
                # only write to clipboard if content changed
                glfw.set_clipboard_string(main_window, user_input.clipboard)

            for button, action, mods in user_input.buttons:
                x, y = glfw.get_cursor_pos(main_window)
                pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
                    main_window, x, y, cached_scale=None
                )
                pos = normalize(pos, g_pool.camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                # Position in img pixels
                pos = denormalize(pos, g_pool.capture.frame_size)

                for plugin in g_pool.plugins:
                    if plugin.on_click(pos, button, action):
                        break

            for key, scancode, action, mods in user_input.keys:
                for plugin in g_pool.plugins:
                    if plugin.on_key(key, scancode, action, mods):
                        break

            for char_ in user_input.chars:
                for plugin in g_pool.plugins:
                    if plugin.on_char(char_):
                        break

            # update screen
            glfw.swap_buffers(main_window)

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal content_scale

            is_minimized = bool(glfw.get_window_attrib(window, glfw.ICONIFIED))

            if is_minimized:
                return

            # Always clear buffers on resize to make sure that there are no overlapping
            # artifacts from previous frames.
            gl_utils.glClear(GL_COLOR_BUFFER_BIT)
            gl_utils.glClearColor(0, 0, 0, 1)

            active_window = glfw.get_current_context()
            glfw.make_context_current(window)
            content_scale = gl_utils.get_content_scale(window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(window)
            g_pool.gui.scale = content_scale
            window_size = w, h
            g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = content_scale
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.make_context_current(active_window)

            # Minimum window size required, otherwise parts of the UI can cause openGL
            # issues with permanent effects. Depends on the content scale, which can
            # potentially be dynamically modified, so we re-adjust the size limits every
            # time here.
            min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale)
            glfw.set_window_size_limits(
                window,
                min_size,
                min_size,
                glfw.DONT_CARE,
                glfw.DONT_CARE,
            )

            # Needed, to update the window buffer while resizing
            consume_events_and_render_buffer()

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
                window, x, y, cached_scale=None
            )
            g_pool.gui.update_mouse(x, y)

            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            if g_pool.flip:
                pos = 1 - pos[0], 1 - pos[1]
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)

            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, paths):
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
        )
        if parse_version(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        camera_is_physically_flipped = eye_id == 0
        g_pool.iconified = False
        g_pool.capture = None
        g_pool.flip = session_settings.get("flip", camera_is_physically_flipped)
        g_pool.display_mode = session_settings.get("display_mode", "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image": "Raw eye camera image. This uses the least amount of CPU power",
            "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.init()
        glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
        if hide_ui:
            glfw.window_hint(glfw.VISIBLE, 0)  # hide window
        title = "Pupil Capture - eye {}".format(eye_id)

        # Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions.
        # As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3
        # cameras, we adjust the default eye window size to a 1:1 content aspect ratio.
        # The size of 500 was chosen s.t. the menu still fits.
        default_window_size = 500 + icon_bar_width, 500
        width, height = session_settings.get("window_size", default_window_size)

        main_window = glfw.create_window(width, height, title, None, None)

        window_position_manager = gl_utils.WindowPositionManager()
        window_pos = window_position_manager.new_window_position(
            window=main_window,
            default_position=window_position_default,
            previous_position=session_settings.get("window_position", None),
        )
        glfw.set_window_pos(main_window, window_pos[0], window_pos[1])

        glfw.make_context_current(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")

        def set_window_size():
            # Get current capture frame size
            f_width, f_height = g_pool.capture.frame_size
            # Eye camera resolutions are too small to be used as default window sizes.
            # We use double their size instead.
            frame_scale_factor = 2
            f_width *= frame_scale_factor
            f_height *= frame_scale_factor

            # Get current display scale factor
            content_scale = gl_utils.get_content_scale(main_window)
            framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
            display_scale_factor = content_scale / framebuffer_scale

            # Scale the capture frame size by display scale factor
            f_width *= display_scale_factor
            f_height *= display_scale_factor

            # Increas the width to account for the added scaled icon bar width
            f_width += icon_bar_width * display_scale_factor

            # Set the newly calculated size (scaled capture frame size + scaled icon bar width)
            glfw.set_window_size(main_window, int(f_width), int(f_height))

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            )
        )
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode]
        )

        general_settings.append(g_pool.display_mode_info)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        plugins_to_load = session_settings.get("loaded_plugins", default_plugins)
        if overwrite_cap_settings:
            # Ensure that overwrite_cap_settings takes preference over source plugins
            # with incorrect settings that were loaded from session settings.
            plugins_to_load.append(overwrite_cap_settings)

        # Add runtime plugins to the list of plugins to load with default arguments,
        # if not already restored from session settings
        plugins_to_load_names = set(name for name, _ in plugins_to_load)
        for runtime_detector in runtime_detectors:
            runtime_name = runtime_detector.__name__
            if runtime_name not in plugins_to_load_names:
                plugins_to_load.append((runtime_name, {}))

        g_pool.plugins = Plugin_List(g_pool, plugins_to_load)

        if not g_pool.capture:
            # Make sure we always have a capture running. Important if there was no
            # capture stored in session settings.
            g_pool.plugins.add(
                g_pool.plugin_by_name[default_capture_name], default_capture_settings
            )

        toggle_general_settings(True)

        g_pool.writer = None
        g_pool.rec_path = None

        # Register callbacks main_window
        glfw.set_framebuffer_size_callback(main_window, on_resize)
        glfw.set_window_iconify_callback(main_window, on_iconify)
        glfw.set_key_callback(main_window, on_window_key)
        glfw.set_char_callback(main_window, on_window_char)
        glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
        glfw.set_cursor_pos_callback(main_window, on_pos)
        glfw.set_scroll_callback(main_window, on_scroll)
        glfw.set_drop_callback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})
        # If previously selected plugin was not loaded this time, we will have an
        # expanded menubar without any menu selected. We need to ensure the menubar is
        # collapsed in this case.
        if all(submenu.collapsed for submenu in g_pool.menubar.elements):
            g_pool.menubar.collapsed = True

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.get_framebuffer_size(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        if platform.system() == "Darwin":
            # On macOS, calls to glfw.swap_buffers() deliberately take longer in case of
            # occluded windows, based on the swap interval value. This causes an FPS drop
            # and leads to problems when recording. To side-step this behaviour, the swap
            # interval is set to zero.
            #
            # Read more about window occlusion on macOS here:
            # https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html
            glfw.swap_interval(0)

        # Event loop
        window_should_close = False
        while not window_should_close:

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        g_pool.rec_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        start_time_synced = notification["start_time_synced"]
                        logger.info(f"Will save eye video to: {g_pool.rec_path}")
                        video_path = os.path.join(
                            g_pool.rec_path, "eye{}.mp4".format(eye_id)
                        )
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(video_path, start_time_synced)
                        elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = MPEG_Writer(video_path, start_time_synced)
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        else:
                            # TODO: wrap recording logic into plugin
                            g_pool.capture.intrinsics.save(
                                g_pool.rec_path, custom_name=f"eye{eye_id}"
                            )
                        finally:
                            g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify(
                        {
                            "subject": "meta.doc",
                            "actor": "eye{}".format(eye_id),
                            "doc": eye.__doc__,
                        }
                    )
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (
                    subject.startswith("start_eye_plugin")
                    and notification["target"] == g_pool.process
                ):
                    try:
                        g_pool.plugins.add(
                            g_pool.plugin_by_name[notification["name"]],
                            notification.get("args", {}),
                        )
                    except KeyError as err:
                        logger.error(f"Attempt to load unknown plugin: {err}")
                elif (
                    subject.startswith("stop_eye_plugin")
                    and notification["target"] == g_pool.process
                ):
                    try:
                        plugin_to_stop = g_pool.plugin_by_name[notification["name"]]
                    except KeyError as err:
                        logger.error(f"Attempt to load unknown plugin: {err}")
                    else:
                        plugin_to_stop.alive = False
                        g_pool.plugins.clean()

                for plugin in g_pool.plugins:
                    plugin.on_notify(notification)

            event = {}
            for plugin in g_pool.plugins:
                plugin.recent_events(event)

            frame = event.get("frame")
            if frame:
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.format(
                                    type(frame), frame_publish_format
                                )
                            )
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send(
                            {
                                "topic": "frame.eye.{}".format(eye_id),
                                "width": frame.width,
                                "height": frame.height,
                                "index": frame.index,
                                "timestamp": frame.timestamp,
                                "format": frame_publish_format,
                                "__raw_data__": [data],
                            }
                        )

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    try:
                        g_pool.writer.write_video_frame(frame)
                    except NonMonotonicTimestampError as e:
                        logger.error(
                            "Recorder received non-monotonic timestamp!"
                            " Stopping the recording!"
                        )
                        logger.debug(str(e))
                        ipc_socket.notify({"subject": "recording.should_stop"})
                        ipc_socket.notify(
                            {"subject": "recording.should_stop", "remote_notify": "all"}
                        )

                for result in event.get(EVENT_KEY, ()):
                    pupil_socket.send(result)

            # GL drawing
            if window_should_update():
                cpu_graph.update()
                if is_window_visible(main_window):
                    consume_events_and_render_buffer()
                glfw.poll_events()
                window_should_close = glfw.window_should_close(main_window)

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer.release()
            g_pool.writer = None

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        # save session persistent settings
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)

        if not hide_ui:
            glfw.restore_window(main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.get_window_pos(main_window)
            session_window_size = glfw.get_window_size(main_window)
            if 0 not in session_window_size:
                f_width, f_height = session_window_size
                if platform.system() in ("Windows", "Linux"):
                    # Store unscaled window size as the operating system will scale the
                    # windows appropriately during launch on Windows and Linux.
                    f_width, f_height = (
                        f_width / content_scale,
                        f_height / content_scale,
                    )
                session_settings["window_size"] = int(f_width), int(f_height)

        session_settings.close()

        for plugin in g_pool.plugins:
            plugin.alive = False
        g_pool.plugins.clean()

    glfw.destroy_window(main_window)
    g_pool.gui.terminate()
    glfw.terminate()
    logger.info("Process shutting down.")
Ejemplo n.º 15
0
Archivo: main.py Proyecto: elmorg/pupil
def session(rec_dir):

    # Callback functions
    def on_resize(window, w, h):
        g_pool.gui.update_window(w, h)
        g_pool.gui.collect_menus()
        graph.adjust_size(w, h)
        adjust_gl_view(w, h)
        for p in g_pool.plugins:
            p.on_window_resize(window, w, h)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfwGetCursorPos(window)
        pos = normalize(pos, glfwGetWindowSize(window))
        pos = denormalize(
            pos,
            (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * y_scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x]
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '%s'" % new_rec_dir)
                global rec_dir
                rec_dir = new_rec_dir
                glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'%s' is not a valid pupil recording" %
                             new_rec_dir)

    tick = delta_t()

    def get_dt():
        return next(tick)

    video_path = [
        f for f in glob(os.path.join(rec_dir, "world.*"))
        if f[-3:] in ('mp4', 'mkv', 'avi')
    ][0]
    timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    #parse info.csv file
    meta_info_path = os.path.join(rec_dir, "info.csv")
    with open(meta_info_path) as info:
        meta_info = dict(
            ((line.strip().split('\t')) for line in info.readlines()))

    rec_version = read_rec_version(meta_info)
    if rec_version >= VersionFormat('0.5'):
        pass
    elif rec_version >= VersionFormat('0.4'):
        update_recording_0v4_to_current(rec_dir)
    elif rec_version >= VersionFormat('0.3'):
        update_recording_0v3_to_current(rec_dir)
        timestamps_path = os.path.join(rec_dir, "timestamps.npy")

    else:
        logger.Error("This recording is to old. Sorry.")
        return

    # Initialize capture
    cap = File_Capture(video_path, timestamps=timestamps_path)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if session_settings.get("version",
                            VersionFormat('0.0')) < get_version(version_file):
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', (0, 0))
    main_window = glfwCreateWindow(
        width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
        rec_dir.split(os.path.sep)[-1], None, None)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    # load pupil_positions, gaze_positions
    pupil_data = load_object(pupil_data_path)
    pupil_list = pupil_data['pupil_positions']
    gaze_list = pupil_data['gaze_positions']

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.version = get_version(version_file)
    g_pool.capture = cap
    g_pool.timestamps = np.load(timestamps_path)
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.rec_version = rec_version
    g_pool.meta_info = meta_info
    g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
                                                     g_pool.timestamps)
    g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
                                                    g_pool.timestamps)
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
                                 ]  #populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index())
        except FileSeekError:
            pass
        g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() - 2)
        except FileSeekError:
            pass
        g_pool.new_seek = True

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.main_menu = ui.Growing_Menu("Settings",
                                       pos=(-350, 20),
                                       size=(300, 400))
    g_pool.main_menu.append(
        ui.Button("Close Pupil Player",
                  lambda: glfwSetWindowShouldClose(main_window, True)))
    g_pool.main_menu.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=0.75,
                  max=2.5,
                  label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
                                         g_pool.version))
    g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
                                         rec_version))
    g_pool.main_menu.append(
        ui.Selector('Open plugin',
                    selection=user_launchable_plugins,
                    labels=[
                        p.__name__.replace('_', ' ')
                        for p in user_launchable_plugins
                    ],
                    setter=open_plugin,
                    getter=lambda: "Select to load"))
    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.main_menu.append(
        ui.Button(
            'Reset window size', lambda: glfwSetWindowSize(
                main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label='Play',
                                  hotkey=GLFW_KEY_SPACE)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=GLFW_KEY_RIGHT)
    g_pool.backward_button = ui.Thumb('backward',
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=GLFW_KEY_LEFT)
    g_pool.quickbar.extend(
        [g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    #we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Scan_Path', {}),
                       ('Vis_Polyline', {}), ('Vis_Circle', {}),
                       ('Export_Launcher', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.notifications = []
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
                                 system_plugins + previous_plugins)

    for p in g_pool.plugins:
        if p.class_name == 'Trim_Marks':
            g_pool.trim_marks = p
            break

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)
    glfwSetDropCallback(main_window, on_drop)
    #trigger on_resize
    on_resize(main_window, *glfwGetFramebufferSize(main_window))

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = cap.get_timestamp() - .03

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"

    while not glfwWindowShouldClose(main_window):

        #grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame_nowait()
            except EndofVideoFileError:
                #end of video logic: pause at last frame.
                g_pool.play = False
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        #report time between now and the last loop interation
        events['dt'] = get_dt()
        #new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(
            g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(
            g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            #update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts != t:
                dt, ts = t - ts, t
            fps_graph.add(1. / dt)

            g_pool.play_button.status_text = str(frame.index)
        #always update the CPU graph
        cpu_graph.update()

        # notify each plugin if there are new notifactions:
        while g_pool.notifications:
            n = g_pool.notifications.pop(0)
            for p in g_pool.plugins:
                p.on_notify(n)

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfwMakeContextCurrent(main_window)
        make_coord_system_norm_based()
        g_pool.image_tex.update_from_frame(frame)
        g_pool.image_tex.draw()
        make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        graph.push_view()
        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        graph.pop_view()
        g_pool.gui.update()

        #present frames at appropriate speed
        cap.wait(frame)

        glfwSwapBuffers(main_window)
        glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.close()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
Ejemplo n.º 16
0
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url,
        user_dir, version, eye_id, cap_src):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools
    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):

        #logging setup
        import logging
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        logger = logging.getLogger()
        logger.handlers = []
        logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
        # create logger for the context of this function
        logger = logging.getLogger(__name__)

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
        import OpenGL.GL as gl
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, make_coord_system_eye_camera_based
        from ui_roi import UIRoi
        #monitoring
        import psutil
        import math

        # helpers/utils
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
        from av_writer import JPEG_Writer, AV_Writer

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D
        }

        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600, 31 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.timebase = timebase

        # Callback functions
        def on_resize(window, w, h):
            if not g_pool.iconified:
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w, h)
                graph.adjust_size(w, h)
                adjust_gl_view(w, h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_button(window, button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return  # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    pos = denormalize(
                        pos,
                        (frame.width, frame.height))  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                            pos, g_pool.u_r.handle_size + 40,
                            g_pool.u_r.handle_size + 40):
                        return  # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, (frame.width, frame.height))
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
        if session_settings.get("version",
                                VersionFormat('0.0')) < g_pool.version:
            logger.info(
                "Session setting are from older version of this app. I will not use those."
            )
            session_settings.clear()
        # Initialize capture
        cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
        default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
        previous_settings = session_settings.get('capture_settings', None)
        if previous_settings and previous_settings['name'] == cap.name:
            cap.settings = previous_settings
        else:
            cap.settings = default_settings

        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.flip = session_settings.get('flip', False)
        g_pool.display_mode = session_settings.get('display_mode',
                                                   'camera_image')
        g_pool.display_mode_info_text = {
            'camera_image':
            "Raw eye camera image. This uses the least amount of CPU power",
            'roi':
            "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            'algorithm':
            "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."
        }

        g_pool.u_r = UIRoi((cap.frame_size[1], cap.frame_size[0]))
        g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))

        def on_frame_size_change(new_size):
            g_pool.u_r = UIRoi((new_size[1], new_size[0]))

        cap.on_frame_size_change = on_frame_size_change

        writer = None

        pupil_detector_settings = session_settings.get(
            'pupil_detector_settings', None)
        last_pupil_detector = pupil_detectors[session_settings.get(
            'last_pupil_detector', Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,
                                                    pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)

        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s" % eye_id
        width, height = session_settings.get('window_size', cap.frame_size)
        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        glfw.glfwSwapInterval(0)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale', 1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                           pos=(-300, 0),
                                           size=(0, 0),
                                           header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(
            ui.Slider('scale',
                      g_pool.gui,
                      setter=set_scale,
                      step=.05,
                      min=1.,
                      max=2.5,
                      label='Interface Size'))
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, frame.width, frame.height)))
        general_settings.append(
            ui.Switch('flip', g_pool, label='Flip image display'))
        general_settings.append(
            ui.Selector('display_mode',
                        g_pool,
                        setter=set_display_mode_info,
                        selection=['camera_image', 'roi', 'algorithm'],
                        labels=['Camera Image', 'ROI', 'Algorithm'],
                        label="Mode"))
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.sidebar.append(general_settings)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector(
            'pupil_detector',
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Detector_2D, Detector_3D],
            labels=['C++ 2d detector', 'C++ 3d detector'],
            label="Detection method")
        general_settings.append(detector_selector)

        # let detector add its GUI
        g_pool.pupil_detector.init_gui(g_pool.sidebar)
        # let the camera add its GUI
        g_pool.capture.init_gui(g_pool.sidebar)

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_key)
        glfw.glfwSetCharCallback(main_window, on_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = cap.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"

        #create a timer to control window update frequency
        window_update_timer = timer(1 / 60.)

        def window_should_update():
            return next(window_update_timer)

        logger.warning('Process started.')

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification['subject']
                if subject == 'eye_process.should_stop':
                    if notification['eye_id'] == eye_id:
                        break
                elif subject == 'set_detection_mapping_mode':
                    if notification['mode'] == '3d':
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    else:
                        if not isinstance(g_pool.pupil_detector, Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                elif subject == 'recording.started':
                    if notification['record_eye']:
                        record_path = notification['rec_path']
                        raw_mode = notification['compression']
                        logger.info("Will save eye video to: %s" % record_path)
                        timestamps_path = os.path.join(
                            record_path, "eye%s_timestamps.npy" % eye_id)
                        if raw_mode and frame.jpeg_buffer:
                            video_path = os.path.join(record_path,
                                                      "eye%s.mp4" % eye_id)
                            writer = JPEG_Writer(video_path, cap.frame_rate)
                        else:
                            video_path = os.path.join(record_path,
                                                      "eye%s.mp4" % eye_id)
                            writer = AV_Writer(video_path, cap.frame_rate)
                        timestamps = []
                elif subject == 'recording.stopped':
                    if writer:
                        logger.info("Done recording.")
                        writer.release()
                        writer = None
                        np.save(timestamps_path, np.asarray(timestamps))
                        del timestamps
                elif subject.startswith('meta.should_doc'):
                    ipc_socket.notify({
                        'subject': 'meta.doc',
                        'actor': 'eye%i' % eye_id,
                        'doc': eye.__doc__
                    })

            # Get an image from the grabber
            try:
                frame = cap.get_frame()
            except CameraCaptureError:
                logger.error("Capture from Camera Failed. Stopping.")
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                cap.seek_to_frame(0)
                frame = cap.get_frame()

            #update performace graphs
            t = frame.timestamp
            dt, ts = t - ts, t
            try:
                fps_graph.add(1. / dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)

            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(
                frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            pupil_socket.send('pupil.%s' % eye_id, result)

            # GL drawing
            if window_should_update():
                if not g_pool.iconified:
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image', 'roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    window_size = glfw.glfwGetWindowSize(main_window)
                    make_coord_system_pixel_based(
                        (frame.height, frame.width, 3), g_pool.flip)

                    if result['method'] == '3d c++':

                        eye_ball = result['projected_sphere']
                        try:
                            pts = cv2.ellipse2Poly(
                                (int(eye_ball['center'][0]),
                                 int(eye_ball['center'][1])),
                                (int(eye_ball['axes'][0] / 2),
                                 int(eye_ball['axes'][1] / 2)),
                                int(eye_ball['angle']), 0, 360, 8)
                        except ValueError as e:
                            pass
                        else:
                            draw_polyline(
                                pts, 2,
                                RGBA(0., .9, .1, result['model_confidence']))

                    if result['confidence'] > 0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly(
                                (int(result['ellipse']['center'][0]),
                                 int(result['ellipse']['center'][1])),
                                (int(result['ellipse']['axes'][0] / 2),
                                 int(result['ellipse']['axes'][1] / 2)),
                                int(result['ellipse']['angle']), 0, 360, 15)
                            confidence = result[
                                'confidence'] * 0.7  #scale it a little
                            draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
                            draw_points([result['ellipse']['center']],
                                        size=20,
                                        color=RGBA(1., 0., 0., confidence),
                                        sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize(
                )  #detector decides if we visualize or not

        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path, np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = g_pool.version
        session_settings[
            'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        cap.close()
        logger.info("Process shutting down.")
Ejemplo n.º 17
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version):
    # general imports
    from time import sleep
    import logging
    import errno
    from glob import glob
    from time import time
    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:

        # imports
        from file_methods import Persistent_Dict, load_object

        # display
        import glfw
        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils
        # capture
        from video_capture import File_Source, EndofVideoFileError

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation
        from vis_scan_path import Vis_Scan_Path
        from vis_eye_video_overlay import Vis_Eye_Video_Overlay
        from seek_control import Seek_Control
        from video_export_launcher import Video_Export_Launcher
        from offline_surface_tracker import Offline_Surface_Tracker
        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from batch_exporter import Batch_Exporter, Batch_Export
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producers import Gaze_From_Recording, Offline_Calibration
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection

        assert VersionFormat(pyglui_version) >= VersionFormat(
            '1.17'), 'pyglui out of date, please upgrade to newest version'

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, 'plugins'))
        system_plugins = [
            Log_Display, Seek_Control, Plugin_Manager, System_Graphs,
            Batch_Export, System_Timelines
        ]
        user_plugins = [
            Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points,
            Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path,
            Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter,
            Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter,
            Annotation_Player, Log_History, Pupil_From_Recording,
            Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor

            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            for x in range(count):
                new_rec_dir = paths[x].decode('utf-8')
                if is_pupil_rec_dir(new_rec_dir):
                    logger.debug(
                        "Starting new session with '{}'".format(new_rec_dir))
                    ipc_pub.notify({
                        "subject": "player_drop_process.should_start",
                        "rec_dir": new_rec_dir
                    })
                    glfw.glfwSetWindowShouldClose(window, True)
                else:
                    logger.error("'{}' is not a valid pupil recording".format(
                        new_rec_dir))

        tick = delta_t()

        def get_dt():
            return next(tick)

        video_path = [
            f for f in glob(os.path.join(rec_dir, "world.*"))
            if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264',
                                          '.mjpeg')
        ][0]
        pupil_data_path = os.path.join(rec_dir, "pupil_data")

        meta_info = load_meta_info(rec_dir)

        # log info about Pupil Platform and Platform in player.log
        logger.info('Application Version: {}'.format(app_version))
        logger.info('System Info: {}'.format(get_system_info()))

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = Global_Container()
        g_pool.app = 'player'
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        # sets itself to g_pool.capture
        File_Source(g_pool, video_path)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.capture.playback_speed = session_settings.get(
            'playback_speed', 1.)

        width, height = session_settings.get('window_size',
                                             g_pool.capture.frame_size)
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(
            width, height, "Pupil Player: " + meta_info["Recording Name"] +
            " - " + rec_dir.split(os.path.sep)[-1], None, None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1])
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        # load pupil_positions, gaze_positions
        g_pool.pupil_data = load_object(pupil_data_path)
        g_pool.binocular = meta_info.get('Eye Mode',
                                         'monocular') == 'binocular'
        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.
        g_pool.new_seek = True
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            'min_data_confidence', 0.6)

        g_pool.pupil_positions = []
        g_pool.gaze_positions = []
        g_pool.fixations = []

        g_pool.notifications_by_frame = correlate_data(
            g_pool.pupil_data['notifications'], g_pool.timestamps)
        g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps
                                           ]  # populated by producer`
        g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps
                                          ]  # populated by producer
        g_pool.fixations_by_frame = [
            [] for x in g_pool.timestamps
        ]  # populated by the fixation detector plugin

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {'subject': 'min_data_confidence_changed'}
            notification['_notify_time_'] = time() + .8
            g_pool.ipc_pub.notify(notification)

        def open_plugin(plugin):
            if plugin == "Select to load":
                return
            g_pool.plugins.add(plugin)

        def purge_plugins():
            for p in g_pool.plugins:
                if p.__class__ in user_plugins:
                    p.alive = False
            g_pool.plugins.clean()

        def do_export(_):
            export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right
            export_dir = os.path.join(g_pool.rec_dir, 'exports',
                                      '{}-{}'.format(*export_range))
            try:
                os.makedirs(export_dir)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    logger.error("Could not create export dir")
                    raise e
                else:
                    overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting."
                    logger.warning(overwrite_warning.format(*export_range))
            else:
                logger.info('Created export dir at "{}"'.format(export_dir))

            notification = {
                'subject': 'should_export',
                'range': export_range,
                'export_dir': export_dir
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({
                'subject': 'player_process.should_start',
                'rec_dir': rec_dir,
                'delay': 2.
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos='hidden')
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu('User Timelines',
                                                 pos=(0., -150.),
                                                 size=(0., 0.),
                                                 header_pos='headline')
        g_pool.user_timelines.color = RGBA(a=0.)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(
            ui.Button(
                'Reset window size', lambda: glfw.glfwSetWindowSize(
                    main_window, g_pool.capture.frame_size[0], g_pool.capture.
                    frame_size[1])))
        general_settings.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.8, .9, 1., 1.1, 1.2] +
                        list(np.arange(1.5, 5.1, .5)),
                        label='Interface Size'))
        general_settings.append(
            ui.Info_Text('Player Version: {}'.format(g_pool.version)))
        general_settings.append(
            ui.Info_Text('Capture Version: {}'.format(
                meta_info['Capture Software Version'])))
        general_settings.append(
            ui.Info_Text('Data Format Version: {}'.format(
                meta_info['Data Format Version'])))
        general_settings.append(
            ui.Slider('min_data_confidence',
                      g_pool,
                      setter=set_data_confidence,
                      step=.05,
                      min=0.0,
                      max=1.0,
                      label='Confidence threshold'))
        general_settings.append(
            ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed',
                       general_settings,
                       label=chr(0xe8b8),
                       on_val=False,
                       off_val=True,
                       setter=toggle_general_settings,
                       label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb('export',
                                        label=chr(0xe2c5),
                                        getter=lambda: False,
                                        setter=do_export,
                                        hotkey='e',
                                        label_font='pupil_icons')
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}),
                           ('Log_Display', {}), ('Raw_Data_Exporter', {}),
                           ('Vis_Polyline', {}), ('Vis_Circle', {}),
                           ('System_Graphs', {}), ('System_Timelines', {}),
                           ('Video_Export_Launcher', {}),
                           ('Pupil_From_Recording', {}),
                           ('Gaze_From_Recording', {})]
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'start_plugin':
                g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
                                   args=n.get('args', {}))
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({
                    'subject': 'meta.doc',
                    'actor': g_pool.app,
                    'doc': player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            'subject': 'meta.doc',
                            'actor': p.class_name,
                            'doc': p.on_notify.__doc__
                        })

        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # grab new frame
            if g_pool.capture.play or g_pool.new_seek:
                g_pool.new_seek = False
                try:
                    new_frame = g_pool.capture.get_frame()
                except EndofVideoFileError:
                    # end of video logic: pause at last frame.
                    g_pool.capture.play = False
                    logger.warning("end of video")

            frame = new_frame.copy()
            events = {}
            events['frame'] = frame
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events['pupil_positions'] = []
            events['gaze_positions'] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture._recent_frame = frame
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ''
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in user_input.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in user_input.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                glfw.glfwSwapBuffers(main_window)

            # present frames at appropriate speed
            g_pool.capture.wait(frame)
            glfw.glfwPollEvents()

        session_settings['playback_speed'] = g_pool.capture.playback_speed
        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['min_data_confidence'] = g_pool.min_data_confidence
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.capture.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Player crashed with trace:\n{}'.format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'player_process.stopped'})
        sleep(1.0)
Ejemplo n.º 18
0
class Offline_Reference_Surface(Reference_Surface):
    """docstring for Offline_Reference_Surface"""
    def __init__(self,g_pool,name="unnamed",saved_definition=None):
        super(Offline_Reference_Surface, self).__init__(name,saved_definition)
        self.g_pool = g_pool
        self.cache = None
        self.gaze_on_srf = [] # points on surface for realtime feedback display

        self.heatmap_detail = .2
        self.heatmap = None
        self.heatmap_texture = None
        self.metrics_gazecount = None
        self.metrics_texture = None

    #cache fn for offline marker
    def locate_from_cache(self,frame_idx):
        if self.cache == None:
            #no cache available cannot update from cache
            return False
        cache_result = self.cache[frame_idx]
        if cache_result == False:
            #cached data not avaible for this frame
            return False
        elif cache_result == None:
            #cached data shows surface not found:
            self.detected = False
            self.m_from_screen = None
            self.m_to_screen = None
            self.gaze_on_srf = []
            self.detected_markers = 0
            return True
        else:
            self.detected = True
            self.m_from_screen = cache_result['m_from_screen']
            self.m_to_screen =  cache_result['m_to_screen']
            self.detected_markers = cache_result['detected_markers']
            self.gaze_on_srf = self.gaze_on_srf_by_frame_idx(frame_idx,self.m_from_screen)
            return True
        raise Exception("Invalid cache entry. Please report Bug.")


    def update_cache(self,marker_cache,idx=None):
        '''
        compute surface m's and gaze points from cached marker data
        entries are:
            - False: when marker cache entry was False (not yet searched)
            - None: when surface was not found
            - {'m_to_screen':,'m_from_screen':,'detected_markers':,gaze_on_srf}
        '''

        # iterations = 0

        if self.cache == None:
            pass
            # self.init_cache(marker_cache)
        elif idx != None:
            #update single data pt
            self.cache.update(idx,self.answer_caching_request(marker_cache,idx))
        else:
            # update where marker cache is not False but surface cache is still false
            # this happens when the markercache was incomplete when this fn was run before
            for i in range(len(marker_cache)):
                if self.cache[i] == False and marker_cache[i] != False:
                    self.cache.update(i,self.answer_caching_request(marker_cache,i))
                    # iterations +=1
        # return iterations



    def init_cache(self,marker_cache):
        if self.defined:
            logger.debug("Full update of surface '%s' positons cache"%self.name)
            self.cache = Cache_List([self.answer_caching_request(marker_cache,i) for i in xrange(len(marker_cache))],positive_eval_fn=lambda x:  (x!=False) and (x!=None))


    def answer_caching_request(self,marker_cache,frame_index):
        visible_markers = marker_cache[frame_index]
        # cache point had not been visited
        if visible_markers == False:
            return False
        # cache point had been visited
        marker_by_id = dict( [ (m['id'],m) for m in visible_markers] )
        visible_ids = set(marker_by_id.keys())
        requested_ids = set(self.markers.keys())
        overlap = visible_ids & requested_ids
        detected_markers = len(overlap)
        if len(overlap)>=min(2,len(requested_ids)):
            yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] )
            uv = np.array( [self.markers[i].uv_coords for i in overlap] )
            yx.shape=(-1,1,2)
            uv.shape=(-1,1,2)
            m_to_screen,mask = cv2.findHomography(uv,yx)
            m_from_screen,mask = cv2.findHomography(yx,uv)

            return {'m_to_screen':m_to_screen,
                    'm_from_screen':m_from_screen,
                    'detected_markers':len(overlap)}
        else:
            #surface not found
            return None


    def gaze_on_srf_by_frame_idx(self,frame_index,m_from_screen):
        return self._on_srf_by_frame_idx(frame_index,m_from_screen,self.g_pool.gaze_positions_by_frame[frame_index])


    def fixations_on_srf_by_frame_idx(self,frame_index,m_from_screen):
        return self._on_srf_by_frame_idx(frame_index,m_from_screen,self.g_pool.fixations_by_frame[frame_index])


    def _on_srf_by_frame_idx(self,frame_idx,m_from_screen,data_by_frame):
        data_on_srf = []
        for d in data_by_frame:
            pos = np.array([d['norm_pos']]).reshape(1,1,2)
            mapped_pos = cv2.perspectiveTransform(pos , m_from_screen )
            mapped_pos.shape = (2)
            on_srf = bool((0 <= mapped_pos[0] <= 1) and (0 <= mapped_pos[1] <= 1))
            data_on_srf.append( {'norm_pos':(mapped_pos[0],mapped_pos[1]),'on_srf':on_srf,'base':d } )
        return data_on_srf


    def gl_display_heatmap(self):
        if self.heatmap_texture and self.detected:

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.heatmap_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()


    def gl_display_metrics(self):
        if self.metrics_texture and self.detected:


            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.metrics_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()


    #### fns to draw surface in seperate window
    def gl_display_in_window(self,world_tex):
        """
        here we map a selected surface onto a seperate window.
        """
        if self._window and self.detected:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            clear_gl_screen()

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_from_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            world_tex.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()


            if self.heatmap_texture:
                self.heatmap_texture.draw()

            # now lets get recent pupil positions on this surface:
            for gp in self.gaze_on_srf:
                draw_points_norm([gp['norm_pos']],color=RGBA(0.0,0.8,0.5,0.8), size=80)

            glfwSwapBuffers(self._window)
            glfwMakeContextCurrent(active_window)


    def generate_heatmap(self,section):

        if self.cache is None:
            logger.warning('Surface cache is not build yet.')
            return


        x,y = self.real_world_size['x'],self.real_world_size['y']
        x = max(1,int(x))
        y = max(1,int(y))

        filter_size = (int(self.heatmap_detail * x)/2)*2 +1
        std_dev = filter_size /6.
        self.heatmap = np.ones((y,x,4),dtype=np.uint8)
        all_gaze = []

        for frame_idx,c_e in enumerate(self.cache[section]):
            if c_e:
                frame_idx+=section.start
                for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']):
                    all_gaze.append(gp['norm_pos'])

        if not all_gaze:
            logger.warning("No gaze data on surface for heatmap found.")
            all_gaze.append((-1.,-1.))
        all_gaze = np.array(all_gaze)
        all_gaze *= [self.real_world_size['x'],self.real_world_size['y']]
        hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1],
                                            bins=[x,y],
                                            range=[[0, self.real_world_size['x']], [0,self.real_world_size['y']]],
                                            normed=False,
                                            weights=None)


        hist = np.rot90(hist)

        #smoothing..
        hist = cv2.GaussianBlur(hist, (filter_size,filter_size),std_dev)
        maxval = np.amax(hist)
        if maxval:
            scale = 255./maxval
        else:
            scale = 0

        hist = np.uint8( hist*(scale) )

        #colormapping
        c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)

        self.heatmap[:,:,:3] = c_map
        self.heatmap[:,:,3] = 125
        self.heatmap_texture = Named_Texture()
        self.heatmap_texture.update_from_ndarray(self.heatmap)


    def visible_count_in_section(self,section):
        #section is a slice
        #return number of frames where surface is visible.
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return 0
        section_cache = self.cache[section]
        return sum(map(bool,section_cache))

    def gaze_on_srf_in_section(self,section=slice(0,None)):
        #section is a slice
        #return number of gazepoints that are on surface in section
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return []
        gaze_on_srf = []
        for frame_idx,c_e in enumerate(self.cache[section]):
            frame_idx+=section.start
            if c_e:
                gaze_on_srf += [gp for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']) if gp['on_srf']]
        return gaze_on_srf
class Offline_Reference_Surface(Reference_Surface):
    """docstring for Offline_Reference_Surface"""
    def __init__(self,g_pool,name="unnamed",saved_definition=None):
        super(Offline_Reference_Surface, self).__init__(name,saved_definition)
        self.g_pool = g_pool
        self.cache = None
        self.gaze_on_srf = [] # points on surface for realtime feedback display

        self.heatmap_detail = .2
        self.heatmap = None
        self.heatmap_texture = None
        self.metrics_gazecount = None
        self.metrics_texture = None

    #cache fn for offline marker
    def locate_from_cache(self,frame_idx):
        if self.cache == None:
            #no cache available cannot update from cache
            return False
        cache_result = self.cache[frame_idx]
        if cache_result == False:
            #cached data not avaible for this frame
            return False
        elif cache_result == None:
            #cached data shows surface not found:
            self.detected = False
            self.m_from_screen = None
            self.m_to_screen = None
            self.gaze_on_srf = []
            self.detected_markers = 0
            return True
        else:
            self.detected = True
            self.m_from_screen = cache_result['m_from_screen']
            self.m_to_screen =  cache_result['m_to_screen']
            self.detected_markers = cache_result['detected_markers']
            self.gaze_on_srf = self.gaze_on_srf_by_frame_idx(frame_idx,self.m_from_screen)
            return True
        raise Exception("Invalid cache entry. Please report Bug.")


    def update_cache(self,marker_cache,idx=None):
        '''
        compute surface m's and gaze points from cached marker data
        entries are:
            - False: when marker cache entry was False (not yet searched)
            - None: when surface was not found
            - {'m_to_screen':,'m_from_screen':,'detected_markers':,gaze_on_srf}
        '''

        # iterations = 0

        if self.cache == None:
            pass
            # self.init_cache(marker_cache)
        elif idx != None:
            #update single data pt
            self.cache.update(idx,self.answer_caching_request(marker_cache,idx))
        else:
            # update where marker cache is not False but surface cache is still false
            # this happens when the markercache was incomplete when this fn was run before
            for i in range(len(marker_cache)):
                if self.cache[i] == False and marker_cache[i] != False:
                    self.cache.update(i,self.answer_caching_request(marker_cache,i))
                    # iterations +=1
        # return iterations



    def init_cache(self,marker_cache):
        if self.defined:
            logger.debug("Full update of surface '%s' positons cache"%self.name)
            self.cache = Cache_List([self.answer_caching_request(marker_cache,i) for i in xrange(len(marker_cache))],positive_eval_fn=lambda x:  (x!=False) and (x!=None))


    def answer_caching_request(self,marker_cache,frame_index):
        visible_markers = marker_cache[frame_index]
        # cache point had not been visited
        if visible_markers == False:
            return False
        # cache point had been visited
        marker_by_id = dict( [ (m['id'],m) for m in visible_markers] )
        visible_ids = set(marker_by_id.keys())
        requested_ids = set(self.markers.keys())
        overlap = visible_ids & requested_ids
        detected_markers = len(overlap)
        if len(overlap)>=min(2,len(requested_ids)):
            yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] )
            uv = np.array( [self.markers[i].uv_coords for i in overlap] )
            yx.shape=(-1,1,2)
            uv.shape=(-1,1,2)
            m_to_screen,mask = cv2.findHomography(uv,yx)
            m_from_screen,mask = cv2.findHomography(yx,uv)

            return {'m_to_screen':m_to_screen,
                    'm_from_screen':m_from_screen,
                    'detected_markers':len(overlap)}
        else:
            #surface not found
            return None


    def gaze_on_srf_by_frame_idx(self,frame_index,m_from_screen):
        return self._on_srf_by_frame_idx(frame_index,m_from_screen,self.g_pool.gaze_positions_by_frame[frame_index])


    def fixations_on_srf_by_frame_idx(self,frame_index,m_from_screen):
        return self._on_srf_by_frame_idx(frame_index,m_from_screen,self.g_pool.fixations_by_frame[frame_index])


    def _on_srf_by_frame_idx(self,frame_idx,m_from_screen,data_by_frame):
        data_on_srf = []
        for d in data_by_frame:
            pos = np.array([d['norm_pos']]).reshape(1,1,2)
            mapped_pos = cv2.perspectiveTransform(pos , m_from_screen )
            mapped_pos.shape = (2)
            on_srf = bool((0 <= mapped_pos[0] <= 1) and (0 <= mapped_pos[1] <= 1))
            data_on_srf.append( {'norm_pos':(mapped_pos[0],mapped_pos[1]),'on_srf':on_srf,'base':d } )
        return data_on_srf


    def gl_display_heatmap(self):
        if self.heatmap_texture and self.detected:

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.heatmap_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()


    def gl_display_metrics(self):
        if self.metrics_texture and self.detected:


            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.metrics_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()


    #### fns to draw surface in seperate window
    def gl_display_in_window(self,world_tex):
        """
        here we map a selected surface onto a seperate window.
        """
        if self._window and self.detected:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(self._window)
            clear_gl_screen()

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_from_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            world_tex.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()


            if self.heatmap_texture:
                self.heatmap_texture.draw()

            # now lets get recent pupil positions on this surface:
            for gp in self.gaze_on_srf:
                draw_points_norm([gp['norm_pos']],color=RGBA(0.0,0.8,0.5,0.8), size=80)

            glfwSwapBuffers(self._window)
            glfwMakeContextCurrent(active_window)


    def generate_heatmap(self,section):

        if self.cache is None:
            logger.warning('Surface cache is not build yet.')
            return


        x,y = self.real_world_size['x'],self.real_world_size['y']
        x = max(1,int(x))
        y = max(1,int(y))

        filter_size = (int(self.heatmap_detail * x)/2)*2 +1
        std_dev = filter_size /6.
        self.heatmap = np.ones((y,x,4),dtype=np.uint8)
        all_gaze = []

        for frame_idx,c_e in enumerate(self.cache[section]):
            if c_e:
                frame_idx+=section.start
                for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']):
                    all_gaze.append(gp['norm_pos'])

        if not all_gaze:
            logger.warning("No gaze data on surface for heatmap found.")
            all_gaze.append((-1.,-1.))
        all_gaze = np.array(all_gaze)
        all_gaze *= [self.real_world_size['x'],self.real_world_size['y']]
        hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1],
                                            bins=[x,y],
                                            range=[[0, self.real_world_size['x']], [0,self.real_world_size['y']]],
                                            normed=False,
                                            weights=None)


        hist = np.rot90(hist)

        #smoothing..
        hist = cv2.GaussianBlur(hist, (filter_size,filter_size),std_dev)
        maxval = np.amax(hist)
        if maxval:
            scale = 255./maxval
        else:
            scale = 0

        hist = np.uint8( hist*(scale) )

        #colormapping
        c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)

        self.heatmap[:,:,:3] = c_map
        self.heatmap[:,:,3] = 125
        self.heatmap_texture = Named_Texture()
        self.heatmap_texture.update_from_ndarray(self.heatmap)


    def visible_count_in_section(self,section):
        #section is a slice
        #return number of frames where surface is visible.
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return 0
        section_cache = self.cache[section]
        return sum(map(bool,section_cache))

    def gaze_on_srf_in_section(self,section=slice(0,None)):
        #section is a slice
        #return number of gazepoints that are on surface in section
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return []
        gaze_on_srf = []
        for frame_idx,c_e in enumerate(self.cache[section]):
            frame_idx+=section.start
            if c_e:
                gaze_on_srf += [gp for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']) if gp['on_srf']]
        return gaze_on_srf
Ejemplo n.º 20
0
class Offline_Reference_Surface(Reference_Surface):
    """docstring for Offline_Reference_Surface"""
    def __init__(self, g_pool, name="unnamed", saved_definition=None):
        super(Offline_Reference_Surface, self).__init__(name, saved_definition)
        self.g_pool = g_pool
        self.cache = None
        self.gaze_on_srf = [
        ]  # points on surface for realtime feedback display

        self.heatmap_detail = .2
        self.heatmap = None
        self.heatmap_texture = None
        self.metrics_gazecount = None
        self.metrics_texture = None

    #cache fn for offline marker
    def locate_from_cache(self, frame_idx):
        if self.cache == None:
            #no cache available cannot update from cache
            return False
        cache_result = self.cache[frame_idx]
        if cache_result == False:
            #cached data not avaible for this frame
            return False
        elif cache_result == None:
            #cached data shows surface not found:
            self.detected = False
            self.m_from_screen = None
            self.m_to_screen = None
            self.gaze_on_srf = []
            self.detected_markers = 0
            return True
        else:
            self.detected = True
            self.m_from_screen = cache_result['m_from_screen']
            self.m_to_screen = cache_result['m_to_screen']
            self.detected_markers = cache_result['detected_markers']
            self.gaze_on_srf = self.gaze_on_srf_by_frame_idx(
                frame_idx, self.m_from_screen)
            return True
        raise Exception("Invalid cache entry. Please report Bug.")

    def update_cache(self,
                     marker_cache,
                     camera_calibration,
                     min_marker_perimeter,
                     min_id_confidence,
                     idx=None):
        '''
        compute surface m's and gaze points from cached marker data
        entries are:
            - False: when marker cache entry was False (not yet searched)
            - None: when surface was not found
            - {'m_to_screen':,'m_from_screen':,'detected_markers':,gaze_on_srf}
        '''

        # iterations = 0
        if self.cache == None:
            pass
            # self.init_cache(marker_cache)
        elif idx != None:
            #update single data pt
            self.cache.update(
                idx,
                self.answer_caching_request(marker_cache, idx,
                                            camera_calibration,
                                            min_marker_perimeter,
                                            min_id_confidence))
        else:
            # update where marker cache is not False but surface cache is still false
            # this happens when the markercache was incomplete when this fn was run before
            for i in range(len(marker_cache)):
                if self.cache[i] == False and marker_cache[i] != False:
                    self.cache.update(
                        i,
                        self.answer_caching_request(marker_cache, i,
                                                    camera_calibration,
                                                    min_marker_perimeter,
                                                    min_id_confidence))
                    # iterations +=1
        # return iterations

    def init_cache(self, marker_cache, camera_calibration,
                   min_marker_perimeter, min_id_confidence):
        if self.defined:
            logger.debug("Full update of surface '%s' positons cache" %
                         self.name)
            self.cache = Cache_List([
                self.answer_caching_request(
                    marker_cache, i, camera_calibration, min_marker_perimeter,
                    min_id_confidence) for i in xrange(len(marker_cache))
            ],
                                    positive_eval_fn=lambda x:
                                    (x != False) and (x != None))

    def answer_caching_request(self, marker_cache, frame_index,
                               camera_calibration, min_marker_perimeter,
                               min_id_confidence):
        visible_markers = marker_cache[frame_index]
        # cache point had not been visited
        if visible_markers == False:
            return False
        res = self._get_location(visible_markers,
                                 camera_calibration,
                                 min_marker_perimeter,
                                 min_id_confidence,
                                 locate_3d=False)
        if res['detected']:
            return res
        else:
            #surface not found
            return None

    def move_vertex(self, vert_idx, new_pos):
        super(Offline_Reference_Surface, self).move_vertex(vert_idx, new_pos)
        self.cache = None
        self.heatmap = None

    def add_marker(self, marker, visible_markers, camera_calibration,
                   min_marker_perimeter, min_id_confidence):
        super(Offline_Reference_Surface,
              self).add_marker(marker, visible_markers, camera_calibration,
                               min_marker_perimeter, min_id_confidence)
        self.cache = None
        self.heatmap = None

    def remove_marker(self, marker):
        super(Offline_Reference_Surface, self).remove_marker(marker)
        self.cache = None
        self.heatmap = None

    def gaze_on_srf_by_frame_idx(self, frame_index, m_from_screen):
        return self.map_data_to_surface(
            self.g_pool.gaze_positions_by_frame[frame_index], m_from_screen)

    def fixations_on_srf_by_frame_idx(self, frame_index, m_from_screen):
        return self.map_data_to_surface(
            self.g_pool.fixations_by_frame[frame_index], m_from_screen)

    def gl_display_heatmap(self):
        if self.heatmap_texture and self.detected:

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1, -1, 1)  # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.heatmap_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()

    def gl_display_metrics(self):
        if self.metrics_texture and self.detected:

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1, -1, 1)  # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.metrics_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()

    def generate_heatmap(self, section):

        if self.cache is None:
            logger.warning('Surface cache is not build yet.')
            return

        x, y = self.real_world_size['x'], self.real_world_size['y']
        x = max(1, int(x))
        y = max(1, int(y))

        filter_size = (int(self.heatmap_detail * x) / 2) * 2 + 1
        std_dev = filter_size / 6.
        self.heatmap = np.ones((y, x, 4), dtype=np.uint8)
        all_gaze = []

        for frame_idx, c_e in enumerate(self.cache[section]):
            if c_e:
                frame_idx += section.start
                for gp in self.gaze_on_srf_by_frame_idx(
                        frame_idx, c_e['m_from_screen']):
                    if gp['confidence'] >= self.g_pool.min_data_confidence:
                        all_gaze.append(gp['norm_pos'])

        if not all_gaze:
            logger.warning("No gaze data on surface for heatmap found.")
            all_gaze.append((-1., -1.))
        all_gaze = np.array(all_gaze)
        all_gaze *= [self.real_world_size['x'], self.real_world_size['y']]
        hist, xedge, yedge = np.histogram2d(
            all_gaze[:, 0],
            all_gaze[:, 1],
            bins=[x, y],
            range=[[0, self.real_world_size['x']],
                   [0, self.real_world_size['y']]],
            normed=False,
            weights=None)

        hist = np.rot90(hist)

        #smoothing..
        hist = cv2.GaussianBlur(hist, (filter_size, filter_size), std_dev)
        maxval = np.amax(hist)
        if maxval:
            scale = 255. / maxval
        else:
            scale = 0

        hist = np.uint8(hist * (scale))

        #colormapping
        c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)

        self.heatmap[:, :, :3] = c_map
        self.heatmap[:, :, 3] = 125
        self.heatmap_texture = Named_Texture()
        self.heatmap_texture.update_from_ndarray(self.heatmap)

    def visible_count_in_section(self, section):
        #section is a slice
        #return number of frames where surface is visible.
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return 0
        section_cache = self.cache[section]
        return sum(map(bool, section_cache))

    def gaze_on_srf_in_section(self, section=slice(0, None)):
        #section is a slice
        #return number of gazepoints that are on surface in section
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return []
        gaze_on_srf = []
        for frame_idx, c_e in enumerate(self.cache[section]):
            frame_idx += section.start
            if c_e:
                gaze_on_srf += [
                    gp for gp in self.gaze_on_srf_by_frame_idx(
                        frame_idx, c_e['m_from_screen']) if gp['on_srf']
                ]
        return gaze_on_srf
Ejemplo n.º 21
0
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
          user_dir, version):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    # logger.setLevel(logging.DEBUG)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            'subject': 'eye_process.should_start.{}'.format(eye_id),
            'eye_id': eye_id,
            'delay': delay
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            'subject': 'eye_process.should_stop.{}'.format(eye_id),
            'eye_id': eye_id,
            'delay': 0.2
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
        ipc_pub.notify(n)

    try:

        # display
        import glfw
        from version_utils import VersionFormat
        from pyglui import ui, cygl, __version__ as pyglui_version
        assert VersionFormat(pyglui_version) >= VersionFormat(
            '1.9'), 'pyglui out of date, please upgrade to newest version'
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic
        logger.info('Application Version: {}'.format(version))
        logger.info('System Info: {}'.format(get_system_info()))

        import audio

        # trigger pupil detector cpp build:
        import pupil_detectors
        del pupil_detectors

        # Plug-ins
        from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from pupil_remote import Pupil_Remote
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from frame_publisher import Frame_Publisher
        from blink_detection import Blink_Detection
        from video_capture import source_classes, manager_classes, Base_Manager, Base_Source
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from audio_capture import Audio_Capture
        from accuracy_visualizer import Accuracy_Visualizer
        # from saccade_detector import Saccade_Detector
        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay

        # UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == 'Windows':
            scroll_factor = 10.0
            window_position_default = (8, 31)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process they are accesible to all plugins
        g_pool = Global_Container()
        g_pool.app = 'capture'
        g_pool.process = 'world'
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eyes_are_alive = eyes_are_alive

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, 'plugins'))
        user_plugins = [
            Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote,
            Time_Sync, Surface_Tracker, Annotation_Capture, Log_History,
            Fixation_Detector, Blink_Detection, Remote_Recorder,
            Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay
        ]
        system_plugins = [
            Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay,
            Plugin_Manager, System_Graphs
        ] + manager_classes + source_classes
        plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins
        user_plugins += [
            p for p in runtime_plugins
            if not isinstance(p, (Base_Manager, Base_Source,
                                  System_Plugin_Base, Calibration_Plugin,
                                  Gaze_Mapping_Plugin))
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_settings = {
            'preferred_names': [
                "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510",
                "B525", "C525", "C615", "C920", "C930e"
            ],
            'frame_size': (1280, 720),
            'frame_rate':
            30
        }

        default_plugins = [("UVC_Source", default_capture_settings),
                           ('Pupil_Data_Relay', {}), ('UVC_Manager', {}),
                           ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
                           ('Display_Recent_Gaze', {}),
                           ('Screen_Marker_Calibration', {}), ('Recorder', {}),
                           ('Pupil_Remote', {}), ('Plugin_Manager', {}),
                           ('System_Graphs', {})]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor
            hdpi_factor = float(
                glfw.glfwGetFramebufferSize(window)[0] /
                glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode('utf-8') for x in range(count)]
            for p in g_pool.plugins:
                p.on_drop(paths)

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, 'user_settings_world'))
        if VersionFormat(session_settings.get("version",
                                              '0.0')) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.detection_mapping_mode = session_settings.get(
            'detection_mapping_mode', '3d')
        g_pool.active_calibration_plugin = None
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.audio_mode = session_settings.get('audio_mode',
                                                audio.default_audio_mode)

        def handle_notifications(n):
            subject = n['subject']
            if subject == 'set_detection_mapping_mode':
                if n['mode'] == '2d':
                    if ("Vector_Gaze_Mapper"
                            in g_pool.active_gaze_mapping_plugin.class_name):
                        logger.warning(
                            "The gaze mapper is not supported in 2d mode. Please recalibrate."
                        )
                        g_pool.plugins.add(
                            g_pool.plugin_by_name['Dummy_Gaze_Mapper'])
                g_pool.detection_mapping_mode = n['mode']
            elif subject == 'start_plugin':
                g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
                                   args=n.get('args', {}))
            elif subject == 'stop_plugin':
                for p in g_pool.plugins:
                    if p.class_name == n['name']:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == 'eye_process.started':
                n = {
                    'subject': 'set_detection_mapping_mode',
                    'mode': g_pool.detection_mapping_mode
                }
                ipc_pub.notify(n)
            elif subject.startswith('meta.should_doc'):
                ipc_pub.notify({
                    'subject': 'meta.doc',
                    'actor': g_pool.app,
                    'doc': world.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            'subject': 'meta.doc',
                            'actor': p.class_name,
                            'doc': p.on_notify.__doc__
                        })

        # window and gl setup
        glfw.glfwInit()
        width, height = session_settings.get('window_size',
                                             (1280 + icon_bar_width, 720))
        main_window = glfw.glfwCreateWindow(width, height,
                                            "Pupil Capture - World")
        window_pos = session_settings.get('window_position',
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = camera_render_size[0] + \
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \
                glfw.glfwGetFramebufferSize(main_window)[1]
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
            ipc_pub.notify({
                'subject': 'world_process.should_start',
                'delay': 2.
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-400, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos='left')
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos='hidden')
        g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
                                             (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu('General', header_pos='headline')
        general_settings.append(
            ui.Selector('gui_user_scale',
                        g_pool,
                        setter=set_scale,
                        selection=[.6, .8, 1., 1.2, 1.4],
                        label='Interface size'))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings.append(ui.Button('Reset window size',
                                          set_window_size))
        general_settings.append(
            ui.Selector('audio_mode', audio, selection=audio.audio_modes))
        general_settings.append(
            ui.Selector('detection_mapping_mode',
                        g_pool,
                        label='detection & mapping mode',
                        setter=set_detection_mapping_mode,
                        selection=['2d', '3d']))
        general_settings.append(
            ui.Switch('eye0_process',
                      label='Detect eye 0',
                      setter=lambda alive: start_stop_eye(0, alive),
                      getter=lambda: eyes_are_alive[0].value))
        general_settings.append(
            ui.Switch('eye1_process',
                      label='Detect eye 1',
                      setter=lambda alive: start_stop_eye(1, alive),
                      getter=lambda: eyes_are_alive[1].value))

        general_settings.append(
            ui.Info_Text('Capture Version: {}'.format(g_pool.version)))
        general_settings.append(
            ui.Button('Restart with default settings', reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon('collapsed',
                       general_settings,
                       label=chr(0xe8b8),
                       on_val=False,
                       off_val=True,
                       setter=toggle_general_settings,
                       label_font='pupil_icons')
        icon.tooltip = 'General Settings'
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get('loaded_plugins', default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(False)

        # now the we have  aproper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config', {})

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        if session_settings.get('eye1_process_alive', False):
            launch_eye_process(1, delay=0.6)
        if session_settings.get('eye0_process_alive', True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({'subject': 'world_process.started'})
        logger.warning('Process started.')

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            #a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events['dt'] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # send new events to ipc:
            del events['pupil_positions']  # already on the wire
            del events['gaze_positions']  # sent earlier
            if 'frame' in events:
                del events['frame']  # send explicity with frame publisher
            if 'depth_frame' in events:
                del events['depth_frame']
            if 'audio_packets' in events:
                del events['audio_packets']
            del events['dt']  # no need to send this
            for topic, data in events.items():
                assert (isinstance(data, (list, tuple)))
                for d in data:
                    ipc_pub.send(topic, d)

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            if window_should_update() and gl_utils.is_window_visible(
                    main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                unused_elements = g_pool.gui.update()
                for button, action, mods in unused_elements.buttons:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)
                    for p in g_pool.plugins:
                        p.on_click(pos, button, action)

                for key, scancode, action, mods in unused_elements.keys:
                    for p in g_pool.plugins:
                        p.on_key(key, scancode, action, mods)

                for char_ in unused_elements.chars:
                    for p in g_pool.plugins:
                        p.on_char(char_)

                glfw.glfwSwapBuffers(main_window)
            glfw.glfwPollEvents()

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
        session_settings['gui_scale'] = g_pool.gui_user_scale
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(
            main_window)
        session_settings['version'] = str(g_pool.version)
        session_settings['eye0_process_alive'] = eyes_are_alive[0].value
        session_settings['eye1_process_alive'] = eyes_are_alive[1].value
        session_settings[
            'detection_mapping_mode'] = g_pool.detection_mapping_mode
        session_settings['audio_mode'] = audio.audio_mode
        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()

    except:
        import traceback
        trace = traceback.format_exc()
        logger.error('Process Capture crashed with trace:\n{}'.format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({'subject': 'world_process.stopped'})
        sleep(1.0)
Ejemplo n.º 22
0
def player(rec_dir, ipc_pub_url, ipc_sub_url,
           ipc_push_url, user_dir, app_version):
    # general imports
    import logging
    import errno
    from glob import glob
    from copy import deepcopy
    from time import time
    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)



    # imports
    from file_methods import Persistent_Dict, load_object

    # display
    import glfw
    # check versions for our own depedencies as they are fast-changing
    from pyglui import __version__ as pyglui_version

    from pyglui import ui, graph, cygl
    from pyglui.cygl.utils import Named_Texture
    import gl_utils
    # capture
    from video_capture import File_Source, EndofVideoFileError, FileSeekError

    # helpers/utils
    from version_utils import VersionFormat
    from methods import normalize, denormalize, delta_t, get_system_info
    from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info

    # monitoring
    import psutil

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
    from vis_circle import Vis_Circle
    from vis_cross import Vis_Cross
    from vis_polyline import Vis_Polyline
    from vis_light_points import Vis_Light_Points
    from vis_watermark import Vis_Watermark
    from vis_fixation import Vis_Fixation
    from vis_scan_path import Vis_Scan_Path
    from vis_eye_video_overlay import Vis_Eye_Video_Overlay
    from seek_bar import Seek_Bar
    from trim_marks import Trim_Marks
    from video_export_launcher import Video_Export_Launcher
    from offline_surface_tracker import Offline_Surface_Tracker
    from marker_auto_trim_marks import Marker_Auto_Trim_Marks
    from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
    # from manual_gaze_correction import Manual_Gaze_Correction
    from batch_exporter import Batch_Exporter
    from log_display import Log_Display
    from annotations import Annotation_Player
    from raw_data_exporter import Raw_Data_Exporter
    from log_history import Log_History
    from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
    from gaze_producers import Gaze_From_Recording, Offline_Calibration

    assert pyglui_version >= '1.7'

    runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
    system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
    user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
                               Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector,
                               Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher,
                               Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player,
                               Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection,
                               Gaze_From_Recording, Offline_Calibration] + runtime_plugins

    available_plugins = system_plugins + user_launchable_plugins
    name_by_index = [p.__name__ for p in available_plugins]
    plugin_by_name = dict(zip(name_by_index, available_plugins))

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_window_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_window_char(window, char):
        g_pool.gui.update_char(char)

    def on_window_mouse_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y*scroll_factor)

    def on_drop(window, count, paths):
        for x in range(count):
            new_rec_dir = paths[x].decode('utf-8')
            if is_pupil_rec_dir(new_rec_dir):
                logger.debug("Starting new session with '{}'".format(new_rec_dir))
                ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
                glfw.glfwSetWindowShouldClose(window, True)
            else:
                logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))

    tick = delta_t()

    def get_dt():
        return next(tick)

    video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
                  if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
    pupil_data_path = os.path.join(rec_dir, "pupil_data")

    meta_info = load_meta_info(rec_dir)

    # log info about Pupil Platform and Platform in player.log
    logger.info('Application Version: {}'.format(app_version))
    logger.info('System Info: {}'.format(get_system_info()))

    # create container for globally scoped vars
    g_pool = Global_Container()
    g_pool.app = 'player'
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url

    # Initialize capture
    cap = File_Source(g_pool, video_path)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
    if VersionFormat(session_settings.get("version", '0.0')) != app_version:
        logger.info("Session setting are a different version of this app. I will not use those.")
        session_settings.clear()

    width, height = session_settings.get('window_size', cap.frame_size)
    window_pos = session_settings.get('window_position', window_position_default)
    glfw.glfwInit()
    main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
                                   + rec_dir.split(os.path.sep)[-1], None, None)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    # load pupil_positions, gaze_positions
    g_pool.pupil_data = load_object(pupil_data_path)
    g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
    g_pool.version = app_version
    g_pool.capture = cap
    g_pool.timestamps = g_pool.capture.timestamps
    g_pool.get_timestamp = lambda: 0.
    g_pool.play = False
    g_pool.new_seek = True
    g_pool.user_dir = user_dir
    g_pool.rec_dir = rec_dir
    g_pool.meta_info = meta_info
    g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)

    g_pool.pupil_positions = []
    g_pool.gaze_positions = []
    g_pool.fixations = []

    g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
    g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
    g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
    g_pool.fixations_by_frame = [[] for x in g_pool.timestamps]  # populated by the fixation detector plugin

    def next_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() + 1)
        except(FileSeekError):
            logger.warning("Could not seek to next frame.")
        else:
            g_pool.new_seek = True

    def prev_frame(_):
        try:
            cap.seek_to_frame(cap.get_frame_index() - 1)
        except(FileSeekError):
            logger.warning("Could not seek to previous frame.")
        else:
            g_pool.new_seek = True

    def toggle_play(new_state):
        if cap.get_frame_index() >= cap.get_frame_count()-5:
            cap.seek_to_frame(1)  # avoid pause set by hitting trimmark pause.
            logger.warning("End of video - restart at beginning.")
        g_pool.play = new_state

    def set_data_confidence(new_confidence):
        g_pool.min_data_confidence = new_confidence
        notification = {'subject': 'min_data_confidence_changed'}
        notification['_notify_time_'] = time()+.8
        g_pool.ipc_pub.notify(notification)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def purge_plugins():
        for p in g_pool.plugins:
            if p.__class__ in user_launchable_plugins:
                p.alive = False
        g_pool.plugins.clean()

    def do_export(_):
        export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
        export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
        try:
            os.makedirs(export_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.error("Could not create export dir")
                raise e
            else:
                overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
                logger.warning(overwrite_warning.format(*export_range))
        else:
            logger.info('Created export dir at "{}"'.format(export_dir))

        notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
        g_pool.ipc_pub.notify(notification)

    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560))
    g_pool.main_menu.append(ui.Button('Reset window size',
                                      lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
    g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
    g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
    g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
    g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
    g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
                                      step=.05, min=0.0, max=1.0, label='Confidence threshold'))

    g_pool.main_menu.append(ui.Info_Text('Open plugins'))

    selector_label = "Select to load"

    def append_selector(label, plugins):
        plugins.sort(key=lambda p: p.__name__)
        plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
        g_pool.main_menu.append(ui.Selector(label,
                                            selection=[selector_label] + plugins,
                                            labels=[selector_label] + plugin_labels,
                                            setter=open_plugin,
                                            getter=lambda: selector_label))

    base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base]
    base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
    launchable = user_launchable_plugins.copy()
    for base_class, label in zip(base_plugins, base_labels):
        member_plugins = []
        for p in user_launchable_plugins:
            if issubclass(p, base_class):
                member_plugins.append(p)
                launchable.remove(p)
        append_selector(label, member_plugins)

    # launchable only contains plugins that could not be assigned to any of the above categories
    append_selector('Other', launchable)

    g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
    g_pool.play_button = ui.Thumb('play',
                                  g_pool,
                                  label=chr(0xf04b),
                                  setter=toggle_play,
                                  hotkey=glfw.GLFW_KEY_SPACE,
                                  label_font='fontawesome',
                                  label_offset_x=5,
                                  label_offset_y=0,
                                  label_offset_size=-24)
    g_pool.play_button.on_color[:] = (0, 1., .0, .8)
    g_pool.forward_button = ui.Thumb('forward',
                                     label=chr(0xf04e),
                                     getter=lambda: False,
                                     setter=next_frame,
                                     hotkey=glfw.GLFW_KEY_RIGHT,
                                     label_font='fontawesome',
                                     label_offset_x=5,
                                     label_offset_y=0,
                                     label_offset_size=-24)
    g_pool.backward_button = ui.Thumb('backward',
                                      label=chr(0xf04a),
                                      getter=lambda: False,
                                      setter=prev_frame,
                                      hotkey=glfw.GLFW_KEY_LEFT,
                                      label_font='fontawesome',
                                      label_offset_x=-5,
                                      label_offset_y=0,
                                      label_offset_size=-24)
    g_pool.export_button = ui.Thumb('export',
                                    label=chr(0xf063),
                                    getter=lambda: False,
                                    setter=do_export,
                                    hotkey='e',
                                    label_font='fontawesome',
                                    label_offset_x=0,
                                    label_offset_y=2,
                                    label_offset_size=-24)
    g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
    g_pool.gui.append(g_pool.quickbar)
    g_pool.gui.append(g_pool.main_menu)

    # we always load these plugins
    system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
    default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}),
                       ('Vis_Circle', {}), ('Video_Export_Launcher', {}),
                       ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
    previous_plugins = session_settings.get('loaded_plugins', default_plugins)
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)


    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetKeyCallback(main_window, on_window_key)
    glfw.glfwSetCharCallback(main_window, on_window_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)
    glfw.glfwSetDropCallback(main_window, on_drop)

    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = None

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 110)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 110)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f REC FPS"

    pupil_graph = graph.Bar_Graph(max_val=1.0)
    pupil_graph.pos = (260, 110)
    pupil_graph.update_rate = 5
    pupil_graph.label = "Confidence: %0.2f"
    g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]

    # trigger on_resize
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'start_plugin':
            g_pool.plugins.add(
                plugin_by_name[n['name']], args=n.get('args', {}))
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({'subject': 'meta.doc',
                            'actor': g_pool.app,
                            'doc': player.__doc__})
            for p in g_pool.plugins:
                if (p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify):
                    ipc_pub.notify({'subject': 'meta.doc',
                                    'actor': p.class_name,
                                    'doc': p.on_notify.__doc__})

    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            for p in g_pool.plugins:
                p.on_notify(n)

        # grab new frame
        if g_pool.play or g_pool.new_seek:
            g_pool.new_seek = False
            try:
                new_frame = cap.get_frame()
            except EndofVideoFileError:
                # end of video logic: pause at last frame.
                g_pool.play = False
                logger.warning("end of video")
            update_graph = True
        else:
            update_graph = False

        frame = new_frame.copy()
        events = {}
        events['frame'] = frame
        # report time between now and the last loop interation
        events['dt'] = get_dt()
        # new positons we make a deepcopy just like the image is a copy.
        events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
        events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])

        if update_graph:
            # update performace graphs
            for p in events['pupil_positions']:
                pupil_graph.add(p['confidence'])

            t = new_frame.timestamp
            if ts and ts != t:
                dt, ts = t-ts, t
                fps_graph.add(1./dt)
            else:
                ts = new_frame.timestamp

            g_pool.play_button.status_text = str(frame.index)
        # always update the CPU graph
        cpu_graph.update()

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        gl_utils.make_coord_system_norm_based()
        g_pool.image_tex.update_from_ndarray(frame.bgr)
        g_pool.image_tex.draw()
        gl_utils.make_coord_system_pixel_based(frame.img.shape)
        # render visual feedback from loaded plugins
        for p in g_pool.plugins:
            p.gl_display()

        fps_graph.draw()
        cpu_graph.draw()
        pupil_graph.draw()
        unused_elements = g_pool.gui.update()
        for b in unused_elements.buttons:
            button, action, mods = b
            pos = glfw.glfwGetCursorPos(main_window)
            pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
            pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
            for p in g_pool.plugins:
                p.on_click(pos, button, action)

        for key, scancode, action, mods in unused_elements.keys:
            for p in g_pool.plugins:
                p.on_key(key, scancode, action, mods)

        for char_ in unused_elements.chars:
            for p in g_pool.plugins:
                p.on_char(char_)

        # present frames at appropriate speed
        cap.wait(frame)

        glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['min_data_confidence'] = g_pool.min_data_confidence
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    cap.cleanup()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'player_process.stopped'})
Ejemplo n.º 23
0
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """

    # modify the root logger for this process
    logger = logging.getLogger()
    # remove inherited handlers
    logger.handlers = []
    # create file handler which logs even debug messages
    fh = logging.FileHandler(os.path.join(g_pool.user_dir,
                                          'eye%s.log' % eye_id),
                             mode='w')
    # fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logger.level + 10)
    # create formatter and add it to the handlers
    formatter = logging.Formatter(
        'Eye' + str(eye_id) +
        ' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    formatter = logging.Formatter(
        'EYE' + str(eye_id) +
        ' Process [%(levelname)s] %(name)s : %(message)s')
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (600, 300 * eye_id)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (600, 31 + 300 * eye_id)
    else:
        scroll_factor = 1.0
        window_position_default = (600, 300 * eye_id)

    # Callback functions
    def on_resize(window, w, h):
        if not g_pool.iconified:
            active_window = glfwGetCurrentContext()
            glfwMakeContextCurrent(window)
            g_pool.gui.update_window(w, h)
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            glfwMakeContextCurrent(active_window)

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_button(window, button, action, mods):
        if g_pool.display_mode == 'roi':
            if action == GLFW_RELEASE and u_r.active_edit_pt:
                u_r.active_edit_pt = False
                return  # if the roi interacts we dont what the gui to interact as well
            elif action == GLFW_PRESS:
                pos = glfwGetCursorPos(window)
                pos = normalize(pos, glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(
                    pos, (frame.width, frame.height))  # Position in img pixels
                if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
                                          u_r.handle_size + 40):
                    return  # if the roi interacts we dont what the gui to interact as well

        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
        g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)

        if u_r.active_edit_pt:
            pos = normalize((x, y), glfwGetWindowSize(main_window))
            if g_pool.flip:
                pos = 1 - pos[0], 1 - pos[1]
            pos = denormalize(pos, (frame.width, frame.height))
            u_r.move_vertex(u_r.active_pt_idx, pos)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    def on_close(window):
        g_pool.quit.value = True
        logger.info('Process closing from window')

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()
    # Initialize capture
    cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
    default_settings = {'frame_size': cap_size, 'frame_rate': 30}
    previous_settings = session_settings.get('capture_settings', None)
    if previous_settings and previous_settings['name'] == cap.name:
        cap.settings = previous_settings
    else:
        cap.settings = default_settings

    # Test capture
    try:
        frame = cap.get_frame()
    except CameraCaptureError:
        logger.error("Could not retrieve image from capture")
        cap.close()
        return

    #signal world that we are ready to go
    pipe_to_world.send('eye%s process ready' % eye_id)

    # any object we attach to the g_pool object *from now on* will only be visible to this process!
    # vars should be declared here to make them visible to the code reader.
    g_pool.iconified = False
    g_pool.capture = cap
    g_pool.flip = session_settings.get('flip', False)
    g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
    g_pool.display_mode_info_text = {
        'camera_image':
        "Raw eye camera image. This uses the least amount of CPU power",
        'roi':
        "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
        'algorithm':
        "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
    }
    # g_pool.draw_pupil = session_settings.get('draw_pupil',True)

    u_r = UIRoi(frame.img.shape)
    u_r.set(session_settings.get('roi', u_r.get()))

    writer = None

    pupil_detector = Canny_Detector(g_pool)

    # UI callback functions
    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def set_display_mode_info(val):
        g_pool.display_mode = val
        g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

    # Initialize glfw
    glfwInit()
    if g_pool.binocular:
        title = "Binocular eye %s" % eye_id
    else:
        title = 'Eye'
    width, height = session_settings.get('window_size',
                                         (frame.width, frame.height))
    main_window = glfwCreateWindow(width, height, title, None, None)
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfwMakeContextCurrent(main_window)
    cygl_init()

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    g_pool.image_tex.update_from_frame(frame)
    glfwSwapInterval(0)

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-300, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface Size'))
    general_settings.append(
        ui.Button(
            'Reset window size',
            lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('display_mode',
                    g_pool,
                    setter=set_display_mode_info,
                    selection=['camera_image', 'roi', 'algorithm'],
                    labels=['Camera Image', 'ROI', 'Algorithm'],
                    label="Mode"))
    general_settings.append(
        ui.Switch('flip', g_pool, label='Flip image display'))
    g_pool.display_mode_info = ui.Info_Text(
        g_pool.display_mode_info_text[g_pool.display_mode])
    general_settings.append(g_pool.display_mode_info)
    g_pool.sidebar.append(general_settings)
    g_pool.gui.append(g_pool.sidebar)
    # let the camera add its GUI
    g_pool.capture.init_gui(g_pool.sidebar)
    # let detector add its GUI
    pupil_detector.init_gui(g_pool.sidebar)

    # Register callbacks main_window
    glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfwSetWindowCloseCallback(main_window, on_close)
    glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfwSetKeyCallback(main_window, on_key)
    glfwSetCharCallback(main_window, on_char)
    glfwSetMouseButtonCallback(main_window, on_button)
    glfwSetCursorPosCallback(main_window, on_pos)
    glfwSetScrollCallback(main_window, on_scroll)

    #set the last saved window size
    on_resize(main_window, *glfwGetWindowSize(main_window))

    # load last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performance graphs
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = frame.timestamp

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    #create a timer to control window update frequency
    window_update_timer = timer(1 / 60.)

    def window_should_update():
        return next(window_update_timer)

    # Event loop
    while not g_pool.quit.value:
        # Get an image from the grabber
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Capture from Camera Failed. Stopping.")
            break
        except EndofVideoFileError:
            logger.warning("Video File is done. Stopping")
            break

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        ###  RECORDING of Eye Video (on demand) ###
        # Setup variables and lists for recording
        if pipe_to_world.poll():
            command, raw_mode = pipe_to_world.recv()
            if command is not None:
                record_path = command
                logger.info("Will save eye video to: %s" % record_path)
                timestamps_path = os.path.join(record_path,
                                               "eye%s_timestamps.npy" % eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = JPEG_Writer(video_path, cap.frame_rate)
                else:
                    video_path = os.path.join(record_path,
                                              "eye%s.mp4" % eye_id)
                    writer = AV_Writer(video_path, cap.frame_rate)
                timestamps = []
            else:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path, np.asarray(timestamps))
                del timestamps

        if writer:
            writer.write_video_frame(frame)
            timestamps.append(frame.timestamp)

        # pupil ellipse detection
        result = pupil_detector.detect(
            frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
        result['id'] = eye_id
        # stream the result
        g_pool.pupil_queue.put(result)

        # GL drawing
        if window_should_update():
            if not g_pool.iconified:
                glfwMakeContextCurrent(main_window)
                clear_gl_screen()

                # switch to work in normalized coordinate space
                if g_pool.display_mode == 'algorithm':
                    g_pool.image_tex.update_from_ndarray(frame.img)
                elif g_pool.display_mode in ('camera_image', 'roi'):
                    g_pool.image_tex.update_from_ndarray(frame.gray)
                else:
                    pass

                make_coord_system_norm_based(g_pool.flip)
                g_pool.image_tex.draw()
                # switch to work in pixel space
                make_coord_system_pixel_based((frame.height, frame.width, 3),
                                              g_pool.flip)

                if result['confidence'] > 0:
                    if result.has_key('axes'):
                        pts = cv2.ellipse2Poly((int(
                            result['center'][0]), int(result['center'][1])),
                                               (int(result['axes'][0] / 2),
                                                int(result['axes'][1] / 2)),
                                               int(result['angle']), 0, 360,
                                               15)
                        cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
                    cygl_draw_points([result['center']],
                                     size=20,
                                     color=cygl_rgba(1., 0., 0., .5),
                                     sharpness=1.)

                # render graphs
                graph.push_view()
                fps_graph.draw()
                cpu_graph.draw()
                graph.pop_view()

                # render GUI
                g_pool.gui.update()

                #render the ROI
                if g_pool.display_mode == 'roi':
                    u_r.draw(g_pool.gui.scale)

                #update screen
                glfwSwapBuffers(main_window)
            glfwPollEvents()

    # END while running

    # in case eye recording was still runnnig: Save&close
    if writer:
        logger.info("Done recording eye.")
        writer = None
        np.save(timestamps_path, np.asarray(timestamps))

    glfwRestoreWindow(main_window)  #need to do this for windows os
    # save session persistent settings
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['roi'] = u_r.get()
    session_settings['flip'] = g_pool.flip
    session_settings['display_mode'] = g_pool.display_mode
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings.close()

    pupil_detector.cleanup()
    g_pool.gui.terminate()
    glfwDestroyWindow(main_window)
    glfwTerminate()
    cap.close()

    #flushing queue in case world process did not exit gracefully
    while not g_pool.pupil_queue.empty():
        g_pool.pupil_queue.get()
    g_pool.pupil_queue.close()

    logger.debug("Process done")
Ejemplo n.º 24
0
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url,
          ipc_push_url, user_dir, version):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.INFO)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    # display
    import glfw
    from pyglui import ui, graph, cygl, __version__ as pyglui_version
    assert pyglui_version >= '1.7'
    from pyglui.cygl.utils import Named_Texture
    import gl_utils

    # monitoring
    import psutil

    # helpers/utils
    from version_utils import VersionFormat
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t, get_system_info, timer
    from uvc import get_time_monotonic
    logger.info('Application Version: {}'.format(version))
    logger.info('System Info: {}'.format(get_system_info()))

    import audio

    # trigger pupil detector cpp build:
    import pupil_detectors
    del pupil_detectors

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin
    from fixation_detector import Fixation_Detector
    from recorder import Recorder
    from display_recent_gaze import Display_Recent_Gaze
    from time_sync import Time_Sync
    from pupil_remote import Pupil_Remote
    from pupil_groups import Pupil_Groups
    from surface_tracker import Surface_Tracker
    from log_display import Log_Display
    from annotations import Annotation_Capture
    from log_history import Log_History
    from frame_publisher import Frame_Publisher
    from blink_detection import Blink_Detection
    from video_capture import source_classes, manager_classes, Base_Manager
    from pupil_data_relay import Pupil_Data_Relay
    from remote_recorder import Remote_Recorder
    from audio_capture import Audio_Capture
    from accuracy_visualizer import Accuracy_Visualizer

    # UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (0, 0)
    elif platform.system() == 'Windows':
        scroll_factor = 10.0
        window_position_default = (8, 31)
    else:
        scroll_factor = 1.0
        window_position_default = (0, 0)

    # g_pool holds variables for this process they are accesible to all plugins
    g_pool = Global_Container()
    g_pool.app = 'capture'
    g_pool.process = 'world'
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.timebase = timebase
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url
    g_pool.eyes_are_alive = eyes_are_alive

    def get_timestamp():
        return get_time_monotonic() - g_pool.timebase.value
    g_pool.get_timestamp = get_timestamp
    g_pool.get_now = get_time_monotonic

    # manage plugins
    runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins'))
    calibration_plugins += [p for p in runtime_plugins if issubclass(p, Calibration_Plugin)]
    runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Calibration_Plugin)]
    manager_classes += [p for p in runtime_plugins if issubclass(p, Base_Manager)]
    runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Base_Manager)]
    user_launchable_plugins = [Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker,
                               Annotation_Capture, Log_History, Fixation_Detector,
                               Blink_Detection, Remote_Recorder, Accuracy_Visualizer] + runtime_plugins
    system_plugins = [Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay]
    plugin_by_index = (system_plugins + user_launchable_plugins + calibration_plugins
                       + gaze_mapping_plugins + manager_classes + source_classes)
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index, plugin_by_index))

    default_capture_settings = {
        'preferred_names': ["Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)",
                            "C510", "B525", "C525", "C615", "C920", "C930e"],
        'frame_size': (1280, 720),
        'frame_rate': 30
    }

    default_plugins = [("UVC_Source", default_capture_settings),
                       ('Pupil_Data_Relay', {}),
                       ('UVC_Manager', {}),
                       ('Log_Display', {}),
                       ('Dummy_Gaze_Mapper', {}),
                       ('Display_Recent_Gaze', {}),
                       ('Screen_Marker_Calibration', {}),
                       ('Recorder', {}),
                       ('Pupil_Remote', {})]

    # Callback functions
    def on_resize(window, w, h):
        if gl_utils.is_window_visible(window):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            gl_utils.adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_window_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_window_char(window, char):
        g_pool.gui.update_char(char)

    def on_window_mouse_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)

    def on_pos(window, x, y):
        hdpi_factor = float(glfw.glfwGetFramebufferSize(
            window)[0] / glfw.glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    def on_drop(window, count, paths):
        paths = [paths[x].decode('utf-8') for x in range(count)]
        for p in g_pool.plugins:
            p.on_drop(paths)

    tick = delta_t()

    def get_dt():
        return next(tick)

    # load session persistent settings
    session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_world'))
    if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
        logger.info("Session setting are from a different version of this app. I will not use those.")
        session_settings.clear()

    g_pool.iconified = False
    g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '3d')
    g_pool.active_calibration_plugin = None
    g_pool.active_gaze_mapping_plugin = None
    g_pool.capture_manager = None

    audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def launch_eye_process(eye_id, delay=0):
        n = {'subject': 'eye_process.should_start.{}'.format(eye_id),
             'eye_id': eye_id, 'delay': delay}
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id,'delay':0.2}
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
        ipc_pub.notify(n)

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'set_detection_mapping_mode':
            if n['mode'] == '2d':
                if ("Vector_Gaze_Mapper" in
                        g_pool.active_gaze_mapping_plugin.class_name):
                    logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
                    g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
            g_pool.detection_mapping_mode = n['mode']
        elif subject == 'start_plugin':
            g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {}))
        elif subject == 'stop_plugin':
            for p in g_pool.plugins:
                if p.class_name == n['name']:
                    p.alive = False
                    g_pool.plugins.clean()
        elif subject == 'eye_process.started':
            n = {'subject': 'set_detection_mapping_mode',
                 'mode': g_pool.detection_mapping_mode}
            ipc_pub.notify(n)
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({'subject': 'meta.doc',
                            'actor': g_pool.app,
                            'doc': world.__doc__})
            for p in g_pool.plugins:
                if (p.on_notify.__doc__
                        and p.__class__.on_notify != Plugin.on_notify):
                    ipc_pub.notify({'subject': 'meta.doc',
                                    'actor': p.class_name,
                                    'doc': p.on_notify.__doc__})

    # window and gl setup
    glfw.glfwInit()
    width, height = session_settings.get('window_size', (1280, 720))
    main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World")
    window_pos = session_settings.get('window_position', window_position_default)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window

    def set_scale(new_scale):
        g_pool.gui_user_scale = new_scale
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    def reset_restart():
        logger.warning("Resetting all settings and restarting Capture.")
        glfw.glfwSetWindowShouldClose(main_window, True)
        ipc_pub.notify({'subject': 'reset_restart_process.should_start'})

    # setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
    g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(ui.Button('Reset to default settings',reset_restart))
    general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface size'))
    general_settings.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window,g_pool.capture.frame_size[0],g_pool.capture.frame_size[1])) )
    general_settings.append(ui.Selector('audio_mode', audio, selection=audio.audio_modes))
    general_settings.append(ui.Selector('detection_mapping_mode',
                                        g_pool,
                                        label='detection & mapping mode',
                                        setter=set_detection_mapping_mode,
                                        selection=['2d','3d']
                                    ))
    general_settings.append(ui.Switch('eye0_process',
                                        label='Detect eye 0',
                                        setter=lambda alive: start_stop_eye(0,alive),
                                        getter=lambda: eyes_are_alive[0].value
                                    ))
    general_settings.append(ui.Switch('eye1_process',
                                        label='Detect eye 1',
                                        setter=lambda alive: start_stop_eye(1,alive),
                                        getter=lambda: eyes_are_alive[1].value
                                    ))
    selector_label = "Select to load"
    labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins]
    user_launchable_plugins.insert(0, selector_label)
    labels.insert(0, selector_label)
    general_settings.append(ui.Selector('Open plugin',
                                        selection=user_launchable_plugins,
                                        labels=labels,
                                        setter=open_plugin,
                                        getter=lambda: selector_label))

    general_settings.append(ui.Info_Text('Capture Version: {}'.format(g_pool.version)))

    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))

    g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
    g_pool.capture_source_menu.collapsed = True

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.calibration_menu.collapsed = True
    g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')

    g_pool.sidebar.append(general_settings)
    g_pool.sidebar.append(g_pool.capture_selector_menu)
    g_pool.sidebar.append(g_pool.capture_source_menu)
    g_pool.sidebar.append(g_pool.calibration_menu)

    g_pool.gui.append(g_pool.sidebar)
    g_pool.gui.append(g_pool.quickbar)

    # plugins that are loaded based on user settings from previous session
    g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(0,ui.Selector(
                        'active_calibration_plugin',
                        getter=lambda: g_pool.active_calibration_plugin.__class__,
                        selection = calibration_plugins,
                        labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
                        setter= open_plugin,label='Method'
                                ))

    #We add the capture selection menu, after a manager has been added:
    g_pool.capture_selector_menu.insert(0,ui.Selector(
                            'capture_manager',
                            setter    = open_plugin,
                            getter    = lambda: g_pool.capture_manager.__class__,
                            selection = manager_classes,
                            labels    = [b.gui_name for b in manager_classes],
                            label     = 'Manager'
                        ))

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfw.glfwSetKeyCallback(main_window, on_window_key)
    glfw.glfwSetCharCallback(main_window, on_window_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)
    glfw.glfwSetDropCallback(main_window, on_drop)

    # gl_state settings
    gl_utils.basic_gl_setup()
    g_pool.image_tex = Named_Texture()

    # now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    # create a timer to control window update frequency
    window_update_timer = timer(1 / 60)
    def window_should_update():
        return next(window_update_timer)

    # set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = g_pool.get_timestamp()

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil0_graph = graph.Bar_Graph(max_val=1.0)
    pupil0_graph.pos = (260, 130)
    pupil0_graph.update_rate = 5
    pupil0_graph.label = "id0 conf: %0.2f"
    pupil1_graph = graph.Bar_Graph(max_val=1.0)
    pupil1_graph.pos = (380, 130)
    pupil1_graph.update_rate = 5
    pupil1_graph.label = "id1 conf: %0.2f"
    pupil_graphs = pupil0_graph, pupil1_graph
    g_pool.graphs = [cpu_graph, fps_graph, pupil0_graph, pupil1_graph]

    # trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    if session_settings.get('eye1_process_alive', False):
        launch_eye_process(1, delay=0.6)
    if session_settings.get('eye0_process_alive', True):
        launch_eye_process(0, delay=0.3)

    ipc_pub.notify({'subject': 'world_process.started'})
    logger.warning('Process started.')

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            for p in g_pool.plugins:
                p.on_notify(n)


        #a dictionary that allows plugins to post and read events
        events = {}
        # report time between now and the last loop interation
        events['dt'] = get_dt()

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.recent_events(events)

        # check if a plugin need to be destroyed
        g_pool.plugins.clean()

        # update performace graphs
        if 'frame' in events:
            t = events["frame"].timestamp
            dt, ts = t-ts, t
            try:
                fps_graph.add(1./dt)
            except ZeroDivisionError:
                pass
        for p in events["pupil_positions"]:
            pupil_graphs[p['id']].add(p['confidence'])
        cpu_graph.update()

        # send new events to ipc:
        del events['pupil_positions']  # already on the wire
        del events['gaze_positions']  # sent earlier
        if 'frame' in events:
            del events['frame']  # send explicity with frame publisher
        if 'depth_frame' in events:
            del events['depth_frame']
        if 'audio_packets' in events:
            del events['audio_packets']
        del events['dt']  # no need to send this
        for topic, data in events.items():
            assert(isinstance(data, (list, tuple)))
            for d in data:
                ipc_pub.send(topic, d)

        glfw.glfwMakeContextCurrent(main_window)
        # render visual feedback from loaded plugins
        if window_should_update() and gl_utils.is_window_visible(main_window):
            for p in g_pool.plugins:
                p.gl_display()

            for g in g_pool.graphs:
                g.draw()

            unused_elements = g_pool.gui.update()
            for button, action, mods in unused_elements.buttons:
                pos = glfw.glfwGetCursorPos(main_window)
                pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                # Position in img pixels
                pos = denormalize(pos, g_pool.capture.frame_size)
                for p in g_pool.plugins:
                    p.on_click(pos, button, action)

            for key, scancode, action, mods in unused_elements.keys:
                for p in g_pool.plugins:
                    p.on_key(key, scancode, action, mods)

            for char_ in unused_elements.chars:
                for p in g_pool.plugins:
                    p.on_char(char_)

            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui_user_scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = str(g_pool.version)
    session_settings['eye0_process_alive'] = eyes_are_alive[0].value
    session_settings['eye1_process_alive'] = eyes_are_alive[1].value
    session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
    session_settings['audio_mode'] = audio.audio_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()

    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()


    # shut down eye processes:
    stop_eye_process(0)
    stop_eye_process(1)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'world_process.stopped'})
Ejemplo n.º 25
0
class Offline_Reference_Surface(Reference_Surface):
    """docstring for Offline_Reference_Surface"""
    def __init__(self,g_pool,name="unnamed",saved_definition=None):
        super(Offline_Reference_Surface, self).__init__(name,saved_definition)
        self.g_pool = g_pool
        self.cache = None
        self.gaze_on_srf = [] # points on surface for realtime feedback display

        self.heatmap_detail = .2
        self.heatmap = None
        self.heatmap_texture = None
        self.metrics_gazecount = None
        self.metrics_texture = None

    #cache fn for offline marker
    def locate_from_cache(self,frame_idx):
        if self.cache == None:
            #no cache available cannot update from cache
            return False
        cache_result = self.cache[frame_idx]
        if cache_result == False:
            #cached data not avaible for this frame
            return False
        elif cache_result == None:
            #cached data shows surface not found:
            self.detected = False
            self.m_from_screen = None
            self.m_to_screen = None
            self.gaze_on_srf = []
            self.detected_markers = 0
            return True
        else:
            self.detected = True
            self.m_from_screen = cache_result['m_from_screen']
            self.m_to_screen =  cache_result['m_to_screen']
            self.detected_markers = cache_result['detected_markers']
            self.gaze_on_srf = self.gaze_on_srf_by_frame_idx(frame_idx,self.m_from_screen)
            return True
        raise Exception("Invalid cache entry. Please report Bug.")


    def update_cache(self,marker_cache,camera_calibration,min_marker_perimeter,idx=None):
        '''
        compute surface m's and gaze points from cached marker data
        entries are:
            - False: when marker cache entry was False (not yet searched)
            - None: when surface was not found
            - {'m_to_screen':,'m_from_screen':,'detected_markers':,gaze_on_srf}
        '''

        # iterations = 0
        if self.cache == None:
            pass
            # self.init_cache(marker_cache)
        elif idx != None:
            #update single data pt
            self.cache.update(idx,self.answer_caching_request(marker_cache,idx,camera_calibration,min_marker_perimeter))
        else:
            # update where marker cache is not False but surface cache is still false
            # this happens when the markercache was incomplete when this fn was run before
            for i in range(len(marker_cache)):
                if self.cache[i] == False and marker_cache[i] != False:
                    self.cache.update(i,self.answer_caching_request(marker_cache,i,camera_calibration,min_marker_perimeter))
                    # iterations +=1
        # return iterations



    def init_cache(self,marker_cache,camera_calibration,min_marker_perimeter):
        if self.defined:
            logger.debug("Full update of surface '%s' positons cache"%self.name)
            self.cache = Cache_List([self.answer_caching_request(marker_cache,i,camera_calibration,min_marker_perimeter) for i in xrange(len(marker_cache))],positive_eval_fn=lambda x:  (x!=False) and (x!=None))


    def answer_caching_request(self,marker_cache,frame_index,camera_calibration,min_marker_perimeter):
        visible_markers = marker_cache[frame_index]
        # cache point had not been visited
        if visible_markers == False:
            return False
        res = self._get_location(visible_markers,camera_calibration,min_marker_perimeter,locate_3d=False)
        if res['detected']:
            return res
        else:
            #surface not found
            return None

    def move_vertex(self,vert_idx,new_pos):
        super(Offline_Reference_Surface, self).move_vertex(vert_idx,new_pos)
        self.cache = None
        self.heatmap = None

    def add_marker(self,marker,visible_markers,camera_calibration,min_marker_perimeter):
        super(Offline_Reference_Surface, self).add_marker(marker,visible_markers,camera_calibration,min_marker_perimeter)
        self.cache = None
        self.heatmap = None

    def remove_marker(self,marker):
        super(Offline_Reference_Surface, self).remove_marker(marker)
        self.cache = None
        self.heatmap = None

    def gaze_on_srf_by_frame_idx(self,frame_index,m_from_screen):
        return self.map_data_to_surface(self.g_pool.gaze_positions_by_frame[frame_index],m_from_screen)

    def fixations_on_srf_by_frame_idx(self,frame_index,m_from_screen):
        return self.map_data_to_surface(self.g_pool.fixations_by_frame[frame_index],m_from_screen)

    def gl_display_heatmap(self):
        if self.heatmap_texture and self.detected:

            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.heatmap_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()


    def gl_display_metrics(self):
        if self.metrics_texture and self.detected:


            # cv uses 3x3 gl uses 4x4 tranformation matricies
            m = cvmat_to_glmat(self.m_to_screen)

            glMatrixMode(GL_PROJECTION)
            glPushMatrix()
            glLoadIdentity()
            glOrtho(0, 1, 0, 1,-1,1) # gl coord convention

            glMatrixMode(GL_MODELVIEW)
            glPushMatrix()
            #apply m  to our quad - this will stretch the quad such that the ref suface will span the window extends
            glLoadMatrixf(m)

            self.metrics_texture.draw()

            glMatrixMode(GL_PROJECTION)
            glPopMatrix()
            glMatrixMode(GL_MODELVIEW)
            glPopMatrix()



    def generate_heatmap(self,section):

        if self.cache is None:
            logger.warning('Surface cache is not build yet.')
            return


        x,y = self.real_world_size['x'],self.real_world_size['y']
        x = max(1,int(x))
        y = max(1,int(y))

        filter_size = (int(self.heatmap_detail * x)/2)*2 +1
        std_dev = filter_size /6.
        self.heatmap = np.ones((y,x,4),dtype=np.uint8)
        all_gaze = []

        for frame_idx,c_e in enumerate(self.cache[section]):
            if c_e:
                frame_idx+=section.start
                for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']):
                    all_gaze.append(gp['norm_pos'])

        if not all_gaze:
            logger.warning("No gaze data on surface for heatmap found.")
            all_gaze.append((-1.,-1.))
        all_gaze = np.array(all_gaze)
        all_gaze *= [self.real_world_size['x'],self.real_world_size['y']]
        hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1],
                                            bins=[x,y],
                                            range=[[0, self.real_world_size['x']], [0,self.real_world_size['y']]],
                                            normed=False,
                                            weights=None)


        hist = np.rot90(hist)

        #smoothing..
        hist = cv2.GaussianBlur(hist, (filter_size,filter_size),std_dev)
        maxval = np.amax(hist)
        if maxval:
            scale = 255./maxval
        else:
            scale = 0

        hist = np.uint8( hist*(scale) )

        #colormapping
        c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)

        self.heatmap[:,:,:3] = c_map
        self.heatmap[:,:,3] = 125
        self.heatmap_texture = Named_Texture()
        self.heatmap_texture.update_from_ndarray(self.heatmap)


    def visible_count_in_section(self,section):
        #section is a slice
        #return number of frames where surface is visible.
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return 0
        section_cache = self.cache[section]
        return sum(map(bool,section_cache))

    def gaze_on_srf_in_section(self,section=slice(0,None)):
        #section is a slice
        #return number of gazepoints that are on surface in section
        #If cache is not available on frames it is reported as not visible
        if self.cache is None:
            return []
        gaze_on_srf = []
        for frame_idx,c_e in enumerate(self.cache[section]):
            frame_idx+=section.start
            if c_e:
                gaze_on_srf += [gp for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']) if gp['on_srf']]
        return gaze_on_srf
Ejemplo n.º 26
0
def world(
    timebase,
    eye_procs_alive,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    preferred_remote_port,
    hide_ui,
):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events
               that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    # general imports
    from time import sleep
    import logging

    # networking
    import zmq
    import zmq_tools

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    def launch_eye_process(eye_id, delay=0):
        n = {
            "subject": "eye_process.should_start.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": delay,
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {
            "subject": "eye_process.should_stop.{}".format(eye_id),
            "eye_id": eye_id,
            "delay": 0.2,
        }
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {"subject": "set_detection_mapping_mode", "mode": new_mode}
        ipc_pub.notify(n)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # display
        import glfw
        from version_utils import VersionFormat
        from pyglui import ui, cygl, __version__ as pyglui_version

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.24"), "pyglui out of date, please upgrade to newest version"
        from pyglui.cygl.utils import Named_Texture
        import gl_utils

        # helpers/utils
        from file_methods import Persistent_Dict
        from methods import normalize, denormalize, delta_t, get_system_info, timer
        from uvc import get_time_monotonic

        logger.info("Application Version: {}".format(version))
        logger.info("System Info: {}".format(get_system_info()))

        import audio

        # trigger pupil detector cpp build:
        import pupil_detectors

        del pupil_detectors

        # Plug-ins
        from plugin import (
            Plugin,
            System_Plugin_Base,
            Plugin_List,
            import_runtime_plugins,
        )
        from plugin_manager import Plugin_Manager
        from calibration_routines import (
            calibration_plugins,
            gaze_mapping_plugins,
            Calibration_Plugin,
            Gaze_Mapping_Plugin,
        )
        from fixation_detector import Fixation_Detector
        from recorder import Recorder
        from display_recent_gaze import Display_Recent_Gaze
        from time_sync import Time_Sync
        from pupil_remote import Pupil_Remote
        from pupil_groups import Pupil_Groups
        from surface_tracker import Surface_Tracker_Online
        from log_display import Log_Display
        from annotations import Annotation_Capture
        from log_history import Log_History
        from frame_publisher import Frame_Publisher
        from blink_detection import Blink_Detection
        from video_capture import (
            source_classes,
            manager_classes,
            Base_Manager,
            Base_Source,
        )
        from pupil_data_relay import Pupil_Data_Relay
        from remote_recorder import Remote_Recorder
        from audio_capture import Audio_Capture
        from accuracy_visualizer import Accuracy_Visualizer

        # from saccade_detector import Saccade_Detector
        from system_graphs import System_Graphs
        from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
        from hololens_relay import Hololens_Relay
        from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (30, 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (8, 90)
        else:
            scroll_factor = 1.0
            window_position_default = (0, 0)

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process they are accessible to all plugins
        g_pool = SimpleNamespace()
        g_pool.app = "capture"
        g_pool.process = "world"
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.timebase = timebase
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.eye_procs_alive = eye_procs_alive
        g_pool.preferred_remote_port = preferred_remote_port

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # manage plugins
        runtime_plugins = import_runtime_plugins(
            os.path.join(g_pool.user_dir, "plugins"))
        user_plugins = [
            Audio_Capture,
            Pupil_Groups,
            Frame_Publisher,
            Pupil_Remote,
            Time_Sync,
            Surface_Tracker_Online,
            Annotation_Capture,
            Log_History,
            Fixation_Detector,
            Blink_Detection,
            Remote_Recorder,
            Accuracy_Visualizer,
            Camera_Intrinsics_Estimation,
            Hololens_Relay,
            Online_Head_Pose_Tracker,
        ]

        system_plugins = ([
            Log_Display,
            Display_Recent_Gaze,
            Recorder,
            Pupil_Data_Relay,
            Plugin_Manager,
            System_Graphs,
        ] + manager_classes + source_classes)
        plugins = (system_plugins + user_plugins + runtime_plugins +
                   calibration_plugins + gaze_mapping_plugins)
        user_plugins += [
            p for p in runtime_plugins if not isinstance(
                p,
                (
                    Base_Manager,
                    Base_Source,
                    System_Plugin_Base,
                    Calibration_Plugin,
                    Gaze_Mapping_Plugin,
                ),
            )
        ]
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}

        default_capture_settings = {
            "preferred_names": [
                "Pupil Cam1 ID2",
                "Logitech Camera",
                "(046d:081d)",
                "C510",
                "B525",
                "C525",
                "C615",
                "C920",
                "C930e",
            ],
            "frame_size": (1280, 720),
            "frame_rate":
            30,
        }

        default_plugins = [
            ("UVC_Source", default_capture_settings),
            ("Pupil_Data_Relay", {}),
            ("UVC_Manager", {}),
            ("Log_Display", {}),
            ("Dummy_Gaze_Mapper", {}),
            ("Display_Recent_Gaze", {}),
            ("Screen_Marker_Calibration", {}),
            ("Recorder", {}),
            ("Pupil_Remote", {}),
            ("Accuracy_Visualizer", {}),
            ("Plugin_Manager", {}),
            ("System_Graphs", {}),
        ]

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()

            for p in g_pool.plugins:
                p.on_window_resize(window, *camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        tick = delta_t()

        def get_dt():
            return next(tick)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_world"))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", 0.8)
        g_pool.detection_mapping_mode = session_settings.get(
            "detection_mapping_mode", "3d")
        g_pool.active_calibration_plugin = None
        g_pool.active_gaze_mapping_plugin = None
        g_pool.capture = None

        audio.audio_mode = session_settings.get("audio_mode",
                                                audio.default_audio_mode)

        def handle_notifications(noti):
            subject = noti["subject"]
            if subject == "set_detection_mapping_mode":
                if noti["mode"] == "2d":
                    if ("Vector_Gaze_Mapper"
                            in g_pool.active_gaze_mapping_plugin.class_name):
                        logger.warning(
                            "The gaze mapper is not supported in 2d mode. Please recalibrate."
                        )
                        g_pool.plugins.add(
                            g_pool.plugin_by_name["Dummy_Gaze_Mapper"])
                g_pool.detection_mapping_mode = noti["mode"]
            elif subject == "start_plugin":
                try:
                    g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]],
                                       args=noti.get("args", {}))
                except KeyError as err:
                    logger.error(f"Attempt to load unknown plugin: {err}")
            elif subject == "stop_plugin":
                for p in g_pool.plugins:
                    if p.class_name == noti["name"]:
                        p.alive = False
                        g_pool.plugins.clean()
            elif subject == "eye_process.started":
                noti = {
                    "subject": "set_detection_mapping_mode",
                    "mode": g_pool.detection_mapping_mode,
                }
                ipc_pub.notify(noti)
            elif subject == "set_min_calibration_confidence":
                g_pool.min_calibration_confidence = noti["value"]
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify({
                    "subject": "meta.doc",
                    "actor": g_pool.app,
                    "doc": world.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            "subject": "meta.doc",
                            "actor": p.class_name,
                            "doc": p.on_notify.__doc__,
                        })
            elif subject == "world_process.adapt_window_size":
                set_window_size()

        width, height = session_settings.get("window_size",
                                             (1280 + icon_bar_width, 720))

        # window and gl setup
        glfw.glfwInit()
        if hide_ui:
            glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0)  # hide window
        main_window = glfw.glfwCreateWindow(width, height,
                                            "Pupil Capture - World")
        window_pos = session_settings.get("window_position",
                                          window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Capture.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({
                "subject": "world_process.should_start",
                "delay": 2.0
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is opened, the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-400, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
                                             (120, -100))
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.6, 0.8, 1.0, 1.2, 1.4],
                label="Interface size",
            ))

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)
            on_resize(main_window, f_width, f_height)

        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Selector("audio_mode", audio, selection=audio.audio_modes))
        general_settings.append(
            ui.Selector(
                "detection_mapping_mode",
                g_pool,
                label="detection & mapping mode",
                setter=set_detection_mapping_mode,
                selection=["disabled", "2d", "3d"],
            ))
        general_settings.append(
            ui.Switch(
                "eye0_process",
                label="Detect eye 0",
                setter=lambda alive: start_stop_eye(0, alive),
                getter=lambda: eye_procs_alive[0].value,
            ))
        general_settings.append(
            ui.Switch(
                "eye1_process",
                label="Detect eye 1",
                setter=lambda alive: start_stop_eye(1, alive),
                getter=lambda: eye_procs_alive[1].value,
            ))

        general_settings.append(
            ui.Info_Text("Capture Version: {}".format(g_pool.version)))
        general_settings.append(
            ui.Button("Restart with default settings", reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        # plugins that are loaded based on user settings from previous session
        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins))

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        toggle_general_settings(True)

        # now that we have a proper window we can load the last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        # trigger setup of window and gl sizes
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        if session_settings.get("eye1_process_alive", True):
            launch_eye_process(1, delay=0.6)
        if session_settings.get("eye0_process_alive", True):
            launch_eye_process(0, delay=0.3)

        ipc_pub.notify({"subject": "world_process.started"})
        logger.warning("Process started.")

        # Event loop
        while (not glfw.glfwWindowShouldClose(main_window)
               and not process_was_interrupted):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            # a dictionary that allows plugins to post and read events
            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            # "blacklisted" events that were already sent
            del events["pupil"]
            del events["gaze"]
            # delete if exists. More expensive than del, so only use it when key might not exist
            events.pop("annotation", None)

            # send new events to ipc:
            if "frame" in events:
                del events["frame"]  # send explicitly with frame publisher
            if "depth_frame" in events:
                del events["depth_frame"]
            if "audio_packets" in events:
                del events["audio_packets"]
            del events["dt"]  # no need to send this
            for data in events.values():
                assert isinstance(data, (list, tuple))
                for d in data:
                    ipc_pub.send(d)

            glfw.glfwMakeContextCurrent(main_window)
            # render visual feedback from loaded plugins
            glfw.glfwPollEvents()
            if window_should_update() and gl_utils.is_window_visible(
                    main_window):

                gl_utils.glViewport(0, 0, *camera_render_size)
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)
                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipboard is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for button, action, mods in user_input.buttons:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, camera_render_size)
                    # Position in img pixels
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                glfw.glfwSwapBuffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["version"] = str(g_pool.version)
        session_settings["eye0_process_alive"] = eye_procs_alive[0].value
        session_settings["eye1_process_alive"] = eye_procs_alive[1].value
        session_settings[
            "min_calibration_confidence"] = g_pool.min_calibration_confidence
        session_settings[
            "detection_mapping_mode"] = g_pool.detection_mapping_mode
        session_settings["audio_mode"] = audio.audio_mode

        if not hide_ui:
            glfw.glfwRestoreWindow(
                main_window)  # need to do this for windows os
            session_settings["window_position"] = glfw.glfwGetWindowPos(
                main_window)
            session_window_size = glfw.glfwGetWindowSize(main_window)
            if 0 not in session_window_size:
                session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Capture crashed with trace:\n{}".format(trace))

    finally:
        # shut down eye processes:
        stop_eye_process(0)
        stop_eye_process(1)

        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "world_process.stopped"})
        sleep(1.0)
Ejemplo n.º 27
0
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
           app_version, debug):
    # general imports
    from time import sleep
    import logging
    from glob import glob
    from time import time, strftime, localtime

    # networking
    import zmq
    import zmq_tools

    import numpy as np

    # zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=("notify", ))

    # log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    try:
        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        from tasklib.background.patches import IPCLoggingPatch

        IPCLoggingPatch.ipc_push_url = ipc_push_url

        # imports
        from file_methods import Persistent_Dict, next_export_sub_dir

        # display
        import glfw

        # check versions for our own depedencies as they are fast-changing
        from pyglui import __version__ as pyglui_version

        from pyglui import ui, cygl
        from pyglui.cygl.utils import Named_Texture, RGBA
        import gl_utils

        # capture
        from video_capture import File_Source

        # helpers/utils
        from version_utils import VersionFormat
        from methods import normalize, denormalize, delta_t, get_system_info
        import player_methods as pm
        from pupil_recording import PupilRecording
        from csv_utils import write_key_value_file

        # Plug-ins
        from plugin import Plugin, Plugin_List, import_runtime_plugins
        from plugin_manager import Plugin_Manager
        from vis_circle import Vis_Circle
        from vis_cross import Vis_Cross
        from vis_polyline import Vis_Polyline
        from vis_light_points import Vis_Light_Points
        from vis_watermark import Vis_Watermark
        from vis_fixation import Vis_Fixation

        from seek_control import Seek_Control
        from surface_tracker import Surface_Tracker_Offline

        # from marker_auto_trim_marks import Marker_Auto_Trim_Marks
        from fixation_detector import Offline_Fixation_Detector
        from log_display import Log_Display
        from annotations import Annotation_Player
        from raw_data_exporter import Raw_Data_Exporter
        from log_history import Log_History
        from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
        from gaze_producer.gaze_from_recording import GazeFromRecording
        from gaze_producer.gaze_from_offline_calibration import (
            GazeFromOfflineCalibration, )
        from system_graphs import System_Graphs
        from system_timelines import System_Timelines
        from blink_detection import Offline_Blink_Detection
        from audio_playback import Audio_Playback
        from video_export.plugins.imotions_exporter import iMotions_Exporter
        from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
        from video_export.plugins.world_video_exporter import World_Video_Exporter
        from head_pose_tracker.offline_head_pose_tracker import (
            Offline_Head_Pose_Tracker, )
        from video_capture import File_Source
        from video_overlay.plugins import Video_Overlay, Eye_Overlay

        from pupil_recording import (
            assert_valid_recording_type,
            InvalidRecordingException,
        )

        assert VersionFormat(pyglui_version) >= VersionFormat(
            "1.27"), "pyglui out of date, please upgrade to newest version"

        process_was_interrupted = False

        def interrupt_handler(sig, frame):
            import traceback

            trace = traceback.format_stack(f=frame)
            logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
            nonlocal process_was_interrupted
            process_was_interrupted = True

        signal.signal(signal.SIGINT, interrupt_handler)

        runtime_plugins = import_runtime_plugins(
            os.path.join(user_dir, "plugins"))
        system_plugins = [
            Log_Display,
            Seek_Control,
            Plugin_Manager,
            System_Graphs,
            System_Timelines,
            Audio_Playback,
        ]
        user_plugins = [
            Vis_Circle,
            Vis_Fixation,
            Vis_Polyline,
            Vis_Light_Points,
            Vis_Cross,
            Vis_Watermark,
            Eye_Overlay,
            Video_Overlay,
            Offline_Fixation_Detector,
            Offline_Blink_Detection,
            Surface_Tracker_Offline,
            Raw_Data_Exporter,
            Annotation_Player,
            Log_History,
            Pupil_From_Recording,
            Offline_Pupil_Detection,
            GazeFromRecording,
            GazeFromOfflineCalibration,
            World_Video_Exporter,
            iMotions_Exporter,
            Eye_Video_Exporter,
            Offline_Head_Pose_Tracker,
        ] + runtime_plugins

        plugins = system_plugins + user_plugins

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal hdpi_factor
            if w == 0 or h == 0:
                return
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            g_pool.camera_render_size = w - int(
                icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(*window_size)
            g_pool.gui.collect_menus()
            for p in g_pool.plugins:
                p.on_window_resize(window, *g_pool.camera_render_size)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x, y = x * hdpi_factor, y * hdpi_factor
            g_pool.gui.update_mouse(x, y)
            pos = x, y
            pos = normalize(pos, g_pool.camera_render_size)
            # Position in img pixels
            pos = denormalize(pos, g_pool.capture.frame_size)
            for p in g_pool.plugins:
                p.on_pos(pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            for path in paths:
                try:
                    assert_valid_recording_type(path)
                    _restart_with_recording(path)
                    return
                except InvalidRecordingException as err:
                    logger.debug(str(err))

            for plugin in g_pool.plugins:
                if plugin.on_drop(paths):
                    break

        def _restart_with_recording(rec_dir):
            logger.debug("Starting new session with '{}'".format(rec_dir))
            ipc_pub.notify({
                "subject": "player_drop_process.should_start",
                "rec_dir": rec_dir
            })
            glfw.glfwSetWindowShouldClose(g_pool.main_window, True)

        tick = delta_t()

        def get_dt():
            return next(tick)

        recording = PupilRecording(rec_dir)
        meta_info = recording.meta_info

        # log info about Pupil Platform and Platform in player.log
        logger.info("Application Version: {}".format(app_version))
        logger.info("System Info: {}".format(get_system_info()))
        logger.debug(f"Debug flag: {debug}")

        icon_bar_width = 50
        window_size = None
        hdpi_factor = 1.0

        # create container for globally scoped vars
        g_pool = SimpleNamespace()
        g_pool.app = "player"
        g_pool.zmq_ctx = zmq_ctx
        g_pool.ipc_pub = ipc_pub
        g_pool.ipc_pub_url = ipc_pub_url
        g_pool.ipc_sub_url = ipc_sub_url
        g_pool.ipc_push_url = ipc_push_url
        g_pool.plugin_by_name = {p.__name__: p for p in plugins}
        g_pool.camera_render_size = None

        video_path = recording.files().core().world().videos()[0].resolve()
        File_Source(
            g_pool,
            timing="external",
            source_path=video_path,
            buffered_decoding=True,
            fill_gaps=True,
        )

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(user_dir, "user_settings_player"))
        if VersionFormat(session_settings.get("version",
                                              "0.0")) != app_version:
            logger.info(
                "Session setting are a different version of this app. I will not use those."
            )
            session_settings.clear()

        width, height = g_pool.capture.frame_size
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        window_pos = session_settings.get("window_position",
                                          window_position_default)
        window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}"

        glfw.glfwInit()
        main_window = glfw.glfwCreateWindow(width, height, window_name, None,
                                            None)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()
        g_pool.main_window = main_window

        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            window_size = (
                g_pool.camera_render_size[0] +
                int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
                glfw.glfwGetFramebufferSize(main_window)[1],
            )
            logger.warning(icon_bar_width * g_pool.gui_user_scale *
                           hdpi_factor)
            glfw.glfwSetWindowSize(main_window, *window_size)

        g_pool.version = app_version
        g_pool.timestamps = g_pool.capture.timestamps
        g_pool.get_timestamp = lambda: 0.0
        g_pool.user_dir = user_dir
        g_pool.rec_dir = rec_dir
        g_pool.meta_info = meta_info
        g_pool.min_data_confidence = session_settings.get(
            "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT)
        g_pool.min_calibration_confidence = session_settings.get(
            "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT)

        # populated by producers
        g_pool.pupil_positions = pm.PupilDataBisector()
        g_pool.gaze_positions = pm.Bisector()
        g_pool.fixations = pm.Affiliator()
        g_pool.eye_movements = pm.Affiliator()

        def set_data_confidence(new_confidence):
            g_pool.min_data_confidence = new_confidence
            notification = {"subject": "min_data_confidence_changed"}
            notification["_notify_time_"] = time() + 0.8
            g_pool.ipc_pub.notify(notification)

        def do_export(_):
            left_idx = g_pool.seek_control.trim_left
            right_idx = g_pool.seek_control.trim_right
            export_range = left_idx, right_idx + 1  # exclusive range.stop
            export_ts_window = pm.exact_window(g_pool.timestamps,
                                               (left_idx, right_idx))

            export_dir = os.path.join(g_pool.rec_dir, "exports")
            export_dir = next_export_sub_dir(export_dir)

            os.makedirs(export_dir)
            logger.info('Created export dir at "{}"'.format(export_dir))

            export_info = {
                "Player Software Version":
                str(g_pool.version),
                "Data Format Version":
                meta_info.min_player_version,
                "Export Date":
                strftime("%d.%m.%Y", localtime()),
                "Export Time":
                strftime("%H:%M:%S", localtime()),
                "Frame Index Range:":
                g_pool.seek_control.get_frame_index_trim_range_string(),
                "Relative Time Range":
                g_pool.seek_control.get_rel_time_trim_range_string(),
                "Absolute Time Range":
                g_pool.seek_control.get_abs_time_trim_range_string(),
            }
            with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
                write_key_value_file(csv, export_info)

            notification = {
                "subject": "should_export",
                "range": export_range,
                "ts_window": export_ts_window,
                "export_dir": export_dir,
            }
            g_pool.ipc_pub.notify(notification)

        def reset_restart():
            logger.warning("Resetting all settings and restarting Player.")
            glfw.glfwSetWindowShouldClose(main_window, True)
            ipc_pub.notify({"subject": "clear_settings_process.should_start"})
            ipc_pub.notify({
                "subject": "player_process.should_start",
                "rec_dir": rec_dir,
                "delay": 2.0,
            })

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu("Settings",
                                           pos=(-500, 0),
                                           size=(-icon_bar_width, 0),
                                           header_pos="left")
        g_pool.iconbar = ui.Scrolling_Menu("Icons",
                                           pos=(-icon_bar_width, 0),
                                           size=(0, 0),
                                           header_pos="hidden")
        g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
        g_pool.timelines.horizontal_constraint = g_pool.menubar
        g_pool.user_timelines = ui.Timeline_Menu("User Timelines",
                                                 pos=(0.0, -150.0),
                                                 size=(0.0, 0.0),
                                                 header_pos="headline")
        g_pool.user_timelines.color = RGBA(a=0.0)
        g_pool.user_timelines.collapsed = True
        # add container that constaints itself to the seekbar height
        vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
        vert_constr.append(g_pool.user_timelines)
        g_pool.timelines.append(vert_constr)

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(ui.Button("Reset window size",
                                          set_window_size))
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2] +
                list(np.arange(1.5, 5.1, 0.5)),
                label="Interface Size",
            ))
        general_settings.append(
            ui.Info_Text(
                f"Minimum Player Version: {meta_info.min_player_version}"))
        general_settings.append(
            ui.Info_Text(f"Player Version: {g_pool.version}"))
        general_settings.append(
            ui.Info_Text(
                f"Recording Software: {meta_info.recording_software_name}"))
        general_settings.append(
            ui.Info_Text(
                f"Recording Software Version: {meta_info.recording_software_version}"
            ))

        general_settings.append(
            ui.Info_Text(
                "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
            ))
        general_settings.append(
            ui.Slider(
                "min_data_confidence",
                g_pool,
                setter=set_data_confidence,
                step=0.05,
                min=0.0,
                max=1.0,
                label="Minimum data confidence",
            ))

        general_settings.append(
            ui.Button("Restart with default settings", reset_restart))

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)

        user_plugin_separator = ui.Separator()
        user_plugin_separator.order = 0.35
        g_pool.iconbar.append(user_plugin_separator)

        g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
                                             (100, -100))
        g_pool.export_button = ui.Thumb(
            "export",
            label=chr(0xE2C5),
            getter=lambda: False,
            setter=do_export,
            hotkey="e",
            label_font="pupil_icons",
        )
        g_pool.quickbar.extend([g_pool.export_button])
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.timelines)
        g_pool.gui.append(g_pool.iconbar)
        g_pool.gui.append(g_pool.quickbar)

        # we always load these plugins
        default_plugins = [
            ("Plugin_Manager", {}),
            ("Seek_Control", {}),
            ("Log_Display", {}),
            ("Raw_Data_Exporter", {}),
            ("Vis_Polyline", {}),
            ("Vis_Circle", {}),
            ("System_Graphs", {}),
            ("System_Timelines", {}),
            ("World_Video_Exporter", {}),
            ("Pupil_From_Recording", {}),
            ("GazeFromRecording", {}),
            ("Audio_Playback", {}),
        ]

        g_pool.plugins = Plugin_List(
            g_pool, session_settings.get("loaded_plugins", default_plugins))

        # Manually add g_pool.capture to the plugin list
        g_pool.plugins._plugins.append(g_pool.capture)
        g_pool.plugins._plugins.sort(key=lambda p: p.order)
        g_pool.capture.init_ui()

        general_settings.insert(
            -1,
            ui.Text_Input(
                "rel_time_trim_section",
                getter=g_pool.seek_control.get_rel_time_trim_range_string,
                setter=g_pool.seek_control.set_rel_time_trim_range_string,
                label="Relative time range to export",
            ),
        )
        general_settings.insert(
            -1,
            ui.Text_Input(
                "frame_idx_trim_section",
                getter=g_pool.seek_control.get_frame_index_trim_range_string,
                setter=g_pool.seek_control.set_frame_index_trim_range_string,
                label="Frame index range to export",
            ),
        )

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        toggle_general_settings(True)

        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # gl_state settings
        gl_utils.basic_gl_setup()
        g_pool.image_tex = Named_Texture()

        # trigger on_resize
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        def handle_notifications(n):
            subject = n["subject"]
            if subject == "start_plugin":
                g_pool.plugins.add(g_pool.plugin_by_name[n["name"]],
                                   args=n.get("args", {}))
            elif subject.startswith("meta.should_doc"):
                ipc_pub.notify({
                    "subject": "meta.doc",
                    "actor": g_pool.app,
                    "doc": player.__doc__
                })
                for p in g_pool.plugins:
                    if (p.on_notify.__doc__
                            and p.__class__.on_notify != Plugin.on_notify):
                        ipc_pub.notify({
                            "subject": "meta.doc",
                            "actor": p.class_name,
                            "doc": p.on_notify.__doc__,
                        })

        while (not glfw.glfwWindowShouldClose(main_window)
               and not process_was_interrupted):

            # fetch newest notifications
            new_notifications = []
            while notify_sub.new_data:
                t, n = notify_sub.recv()
                new_notifications.append(n)

            # notify each plugin if there are new notifications:
            for n in new_notifications:
                handle_notifications(n)
                for p in g_pool.plugins:
                    p.on_notify(n)

            events = {}
            # report time between now and the last loop interation
            events["dt"] = get_dt()

            # pupil and gaze positions are added by their respective producer plugins
            events["pupil"] = []
            events["gaze"] = []

            # allow each Plugin to do its work.
            for p in g_pool.plugins:
                p.recent_events(events)

            # check if a plugin need to be destroyed
            g_pool.plugins.clean()

            glfw.glfwMakeContextCurrent(main_window)
            glfw.glfwPollEvents()
            # render visual feedback from loaded plugins
            if gl_utils.is_window_visible(main_window):

                gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
                g_pool.capture.gl_display()
                for p in g_pool.plugins:
                    p.gl_display()

                gl_utils.glViewport(0, 0, *window_size)

                try:
                    clipboard = glfw.glfwGetClipboardString(
                        main_window).decode()
                except AttributeError:  # clipbaord is None, might happen on startup
                    clipboard = ""
                g_pool.gui.update_clipboard(clipboard)
                user_input = g_pool.gui.update()
                if user_input.clipboard and user_input.clipboard != clipboard:
                    # only write to clipboard if content changed
                    glfw.glfwSetClipboardString(main_window,
                                                user_input.clipboard.encode())

                for b in user_input.buttons:
                    button, action, mods = b
                    x, y = glfw.glfwGetCursorPos(main_window)
                    pos = x * hdpi_factor, y * hdpi_factor
                    pos = normalize(pos, g_pool.camera_render_size)
                    pos = denormalize(pos, g_pool.capture.frame_size)

                    for plugin in g_pool.plugins:
                        if plugin.on_click(pos, button, action):
                            break

                for key, scancode, action, mods in user_input.keys:
                    for plugin in g_pool.plugins:
                        if plugin.on_key(key, scancode, action, mods):
                            break

                for char_ in user_input.chars:
                    for plugin in g_pool.plugins:
                        if plugin.on_char(char_):
                            break

                # present frames at appropriate speed
                g_pool.seek_control.wait(events["frame"].timestamp)
                glfw.glfwSwapBuffers(main_window)

        session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
        session_settings["min_data_confidence"] = g_pool.min_data_confidence
        session_settings[
            "min_calibration_confidence"] = g_pool.min_calibration_confidence
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["window_position"] = glfw.glfwGetWindowPos(
            main_window)
        session_settings["version"] = str(g_pool.version)

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        # de-init all running plugins
        for p in g_pool.plugins:
            p.alive = False
        g_pool.plugins.clean()

        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)

    except Exception:
        import traceback

        trace = traceback.format_exc()
        logger.error("Process Player crashed with trace:\n{}".format(trace))
    finally:
        logger.info("Process shutting down.")
        ipc_pub.notify({"subject": "player_process.stopped"})
        sleep(1.0)
Ejemplo n.º 28
0
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport
        from ui_roi import UIRoi

        # monitoring
        import psutil

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, AV_Writer
        from ndsi import H264Writer
        from video_capture import source_classes
        from video_capture import manager_classes

        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy

        pupil_detectors = {
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D,
            Detector_Dummy.__name__: Detector_Dummy,
        }

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = "capture"
        g_pool.process = "eye{}".format(eye_id)
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor

            active_window = glfw.glfwGetCurrentContext()
            glfw.glfwMakeContextCurrent(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x *= hdpi_factor
            y *= hdpi_factor
            g_pool.gui.update_mouse(x, y)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, g_pool.capture.frame_size)
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            plugins = (g_pool.capture_manager, g_pool.capture)
            # call `on_drop` callbacks until a plugin indicates
            # that it has consumed the event (by returning True)
            any(p.on_drop(paths) for p in plugins)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
        )
        if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.iconified = False
        g_pool.capture = None
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get("flip", False)
        g_pool.display_mode = session_settings.get("display_mode", "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image": "Raw eye camera image. This uses the least amount of CPU power",
            "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        capture_manager_settings = session_settings.get(
            "capture_manager_settings", ("UVC_Manager", {})
        )

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__: c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](
            g_pool, **manager_settings
        )

        if eye_id == 0:
            cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"]
        else:
            cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"]

        # Initialize capture
        default_settings = (
            "UVC_Source",
            {"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120},
        )

        capture_source_settings = overwrite_cap_settings or session_settings.get(
            "capture_settings", default_settings
        )
        source_class_name, source_settings = capture_source_settings
        source_class_by_name = {c.__name__: c for c in source_classes}
        g_pool.capture = source_class_by_name[source_class_name](
            g_pool, **source_settings
        )
        assert g_pool.capture

        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get("roi")
        if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        pupil_detector_settings = session_settings.get("pupil_detector_settings", None)
        last_pupil_detector = pupil_detectors[
            session_settings.get("last_pupil_detector", Detector_2D.__name__)
        ]
        g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings)

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.deinit_ui()
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_ui()

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.glfwInit()
        title = "Pupil Capture - eye {}".format(eye_id)

        width, height = g_pool.capture.frame_size
        width *= 2
        height *= 2
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get("window_position", window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2],
                label="Interface Size",
            )
        )

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width *= 2
            f_height *= 2
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        def uroi_on_mouse_button(button, action, mods):
            if g_pool.display_mode == "roi":
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    x *= hdpi_factor
                    y *= hdpi_factor
                    pos = normalize((x, y), camera_render_size)
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(
                        pos, g_pool.capture.frame_size
                    )  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                        pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size
                    ):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            )
        )
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode]
        )

        general_settings.append(g_pool.display_mode_info)

        detector_selector = ui.Selector(
            "pupil_detector",
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Detector_Dummy, Detector_2D, Detector_3D],
            labels=["disabled", "C++ 2d detector", "C++ 3d detector"],
            label="Detection method",
        )
        general_settings.append(detector_selector)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)
        toggle_general_settings(False)

        g_pool.pupil_detector.init_ui()
        g_pool.capture.init_ui()
        g_pool.capture_manager.init_ui()
        g_pool.writer = None

        def replace_source(source_class_name, source_settings):
            g_pool.capture.deinit_ui()
            g_pool.capture.cleanup()
            g_pool.capture = source_class_by_name[source_class_name](
                g_pool, **source_settings
            )
            g_pool.capture.init_ui()
            if g_pool.writer:
                logger.info("Done recording.")
                try:
                    g_pool.writer.release()
                except RuntimeError:
                    logger.error("No eye video recorded")
                g_pool.writer = None

        g_pool.replace_source = replace_source  # for ndsi capture

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "set_detection_mapping_mode":
                    if notification["mode"] == "3d":
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    elif notification["mode"] == "2d":
                        if not isinstance(g_pool.pupil_detector, Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                    else:
                        if not isinstance(g_pool.pupil_detector, Detector_Dummy):
                            set_detector(Detector_Dummy)
                        detector_selector.read_only = True
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        record_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        logger.info("Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(
                            record_path, "eye{}.mp4".format(eye_id)
                        )
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(
                                video_path, g_pool.capture.frame_rate
                            )
                        elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = AV_Writer(
                                video_path, g_pool.capture.frame_rate
                            )
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify(
                        {
                            "subject": "meta.doc",
                            "actor": "eye{}".format(eye_id),
                            "doc": eye.__doc__,
                        }
                    )
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (
                    subject.startswith("start_eye_capture")
                    and notification["target"] == g_pool.process
                ):
                    replace_source(notification["name"], notification["args"])
                elif notification["subject"].startswith("pupil_detector.set_property"):
                    target_process = notification.get("target", g_pool.process)
                    should_apply = target_process == g_pool.process

                    if should_apply:
                        try:
                            property_name = notification["name"]
                            property_value = notification["value"]
                            if "2d" in notification["subject"]:
                                g_pool.pupil_detector.set_2d_detector_property(
                                    property_name, property_value
                                )
                            elif "3d" in notification["subject"]:
                                if not isinstance(g_pool.pupil_detector, Detector_3D):
                                    raise ValueError(
                                        "3d properties are only available"
                                        " if 3d detector is active"
                                    )
                                g_pool.pupil_detector.set_3d_detector_property(
                                    property_name, property_value
                                )
                            else:
                                raise KeyError(
                                    "Notification subject does not "
                                    "specifiy detector type."
                                )
                            logger.debug(
                                "`{}` property set to {}".format(
                                    property_name, property_value
                                )
                            )
                        except KeyError:
                            logger.error("Malformed notification received")
                            logger.debug(traceback.format_exc())
                        except (ValueError, TypeError):
                            logger.error("Invalid property or value")
                            logger.debug(traceback.format_exc())
                elif notification["subject"].startswith(
                    "pupil_detector.broadcast_properties"
                ):
                    target_process = notification.get("target", g_pool.process)
                    should_respond = target_process == g_pool.process
                    if should_respond:
                        props = g_pool.pupil_detector.get_detector_properties()
                        properties_broadcast = {
                            "subject": "pupil_detector.properties.{}".format(eye_id),
                            **props,  # add properties to broadcast
                        }
                        ipc_socket.notify(properties_broadcast)
                g_pool.capture.on_notify(notification)
                g_pool.capture_manager.on_notify(notification)

            # Get an image from the grabber
            event = {}
            g_pool.capture.recent_events(event)
            frame = event.get("frame")
            g_pool.capture_manager.recent_events(event)
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
                    f_height,
                    f_width,
                ):
                    g_pool.pupil_detector.on_resolution_change(
                        (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
                        g_pool.capture.frame_size,
                    )
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.format(
                                    type(frame), frame_publish_format
                                )
                            )
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send(
                            {
                                "topic": "frame.eye.{}".format(eye_id),
                                "width": frame.width,
                                "height": frame.height,
                                "index": frame.index,
                                "timestamp": frame.timestamp,
                                "format": frame_publish_format,
                                "__raw_data__": [data],
                            }
                        )

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                # pupil ellipse detection
                result = g_pool.pupil_detector.detect(
                    frame, g_pool.u_r, g_pool.display_mode == "algorithm"
                )
                if result is not None:
                    result["id"] = eye_id
                    result["topic"] = "pupil.{}".format(eye_id)
                    pupil_socket.send(result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    if frame:
                        # switch to work in normalized coordinate space
                        if g_pool.display_mode == "algorithm":
                            g_pool.image_tex.update_from_ndarray(frame.img)
                        elif g_pool.display_mode in ("camera_image", "roi"):
                            g_pool.image_tex.update_from_ndarray(frame.gray)
                        else:
                            pass
                    glViewport(0, 0, *camera_render_size)
                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    f_width, f_height = g_pool.capture.frame_size
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    if frame and result:
                        if result["method"] == "3d c++":
                            eye_ball = result["projected_sphere"]
                            try:
                                pts = cv2.ellipse2Poly(
                                    (
                                        int(eye_ball["center"][0]),
                                        int(eye_ball["center"][1]),
                                    ),
                                    (
                                        int(eye_ball["axes"][0] / 2),
                                        int(eye_ball["axes"][1] / 2),
                                    ),
                                    int(eye_ball["angle"]),
                                    0,
                                    360,
                                    8,
                                )
                            except ValueError as e:
                                pass
                            else:
                                draw_polyline(
                                    pts,
                                    2,
                                    RGBA(0.0, 0.9, 0.1, result["model_confidence"]),
                                )
                        if result["confidence"] > 0:
                            if "ellipse" in result:
                                pts = cv2.ellipse2Poly(
                                    (
                                        int(result["ellipse"]["center"][0]),
                                        int(result["ellipse"]["center"][1]),
                                    ),
                                    (
                                        int(result["ellipse"]["axes"][0] / 2),
                                        int(result["ellipse"]["axes"][1] / 2),
                                    ),
                                    int(result["ellipse"]["angle"]),
                                    0,
                                    360,
                                    15,
                                )
                                confidence = result["confidence"] * 0.7
                                draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence))
                                draw_points(
                                    [result["ellipse"]["center"]],
                                    size=20,
                                    color=RGBA(1.0, 0.0, 0.0, confidence),
                                    sharpness=1.0,
                                )

                    glViewport(0, 0, *camera_render_size)
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == "roi":
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    glViewport(0, 0, *window_size)
                    make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    unused_elements = g_pool.gui.update()
                    for butt in unused_elements.buttons:
                        uroi_on_mouse_button(*butt)

                    make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)

                    g_pool.pupil_detector.visualize()  # detector decides if we visualize or not

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer = None

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        # save session persistent settings
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["roi"] = g_pool.u_r.get()
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["capture_settings"] = (
            g_pool.capture.class_name,
            g_pool.capture.get_init_dict(),
        )
        session_settings["capture_manager_settings"] = (
            g_pool.capture_manager.class_name,
            g_pool.capture_manager.get_init_dict(),
        )
        session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
        session_settings["version"] = str(g_pool.version)
        session_settings[
            "last_pupil_detector"
        ] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            "pupil_detector_settings"
        ] = g_pool.pupil_detector.get_settings()

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        g_pool.capture.deinit_ui()
        g_pool.capture_manager.deinit_ui()
        g_pool.pupil_detector.deinit_ui()

        g_pool.pupil_detector.cleanup()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()

        glfw.glfwDestroyWindow(main_window)
        g_pool.gui.terminate()
        glfw.glfwTerminate()
        logger.info("Process shutting down.")
Ejemplo n.º 29
0
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
          user_dir, version):
    """Reads world video and runs plugins.

    Creates a window, gl context.
    Grabs images from a capture.
    Maps pupil to gaze data
    Can run various plug-ins.

    Reacts to notifications:
        ``set_detection_mapping_mode``
        ``eye_process.started``
        ``start_plugin``

    Emits notifications:
        ``eye_process.should_start``
        ``eye_process.should_stop``
        ``set_detection_mapping_mode``
        ``world_process.started``
        ``world_process.stopped``
        ``recording.should_stop``: Emits on camera failure
        ``launcher_process.should_stop``

    Emits data:
        ``gaze``: Gaze data from current gaze mapping plugin.``
        ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
    """

    # We defer the imports because of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    # This is not harmful but unnecessary.

    #general imports
    from time import time, sleep
    import numpy as np
    import logging

    #networking
    import zmq
    import zmq_tools

    #zmq ipc setup
    zmq_ctx = zmq.Context()
    ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                       ipc_sub_url,
                                       topics=('pupil', ))
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
                                        ipc_sub_url,
                                        topics=('notify', ))

    #log setup
    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    #display
    import glfw
    from pyglui import ui, graph, cygl, __version__ as pyglui_version
    assert pyglui_version >= '1.0'
    from pyglui.cygl.utils import Named_Texture
    from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush, is_window_visible

    #monitoring
    import psutil

    # helpers/utils
    from version_utils import VersionFormat
    from file_methods import Persistent_Dict
    from methods import normalize, denormalize, delta_t, get_system_info
    from uvc import get_time_monotonic
    logger.info('Application Version: %s' % version)
    logger.info('System Info: %s' % get_system_info())

    # video sources
    from video_capture import InitialisationError, StreamError, Fake_Source, EndofVideoFileError, source_classes, manager_classes
    source_by_name = {src.class_name(): src for src in source_classes}
    import audio

    #trigger pupil detector cpp build:
    import pupil_detectors
    del pupil_detectors

    # Plug-ins
    from plugin import Plugin, Plugin_List, import_runtime_plugins
    from calibration_routines import calibration_plugins, gaze_mapping_plugins
    from fixation_detector import Fixation_Detector_3D
    from recorder import Recorder
    from show_calibration import Show_Calibration
    from display_recent_gaze import Display_Recent_Gaze
    from time_sync import Time_Sync
    from pupil_remote import Pupil_Remote
    from pupil_groups import Pupil_Groups
    from surface_tracker import Surface_Tracker
    from log_display import Log_Display
    from annotations import Annotation_Capture
    from log_history import Log_History
    from frame_publisher import Frame_Publisher

    #UI Platform tweaks
    if platform.system() == 'Linux':
        scroll_factor = 10.0
        window_position_default = (0, 0)
    elif platform.system() == 'Windows':
        scroll_factor = 1.0
        window_position_default = (8, 31)
    else:
        scroll_factor = 1.0
        window_position_default = (0, 0)

    #g_pool holds variables for this process they are accesible to all plugins
    g_pool = Global_Container()
    g_pool.app = 'capture'
    g_pool.user_dir = user_dir
    g_pool.version = version
    g_pool.timebase = timebase
    g_pool.zmq_ctx = zmq_ctx
    g_pool.ipc_pub = ipc_pub
    g_pool.ipc_pub_url = ipc_pub_url
    g_pool.ipc_sub_url = ipc_sub_url
    g_pool.ipc_push_url = ipc_push_url
    g_pool.eyes_are_alive = eyes_are_alive

    def get_timestamp():
        return get_time_monotonic() - g_pool.timebase.value

    g_pool.get_timestamp = get_timestamp
    g_pool.get_now = get_time_monotonic

    #manage plugins
    runtime_plugins = import_runtime_plugins(
        os.path.join(g_pool.user_dir, 'plugins'))
    user_launchable_plugins = [
        Pupil_Groups, Frame_Publisher, Show_Calibration, Pupil_Remote,
        Time_Sync, Surface_Tracker, Annotation_Capture, Log_History,
        Fixation_Detector_3D
    ] + runtime_plugins
    system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
    plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes
    name_by_index = [p.__name__ for p in plugin_by_index]
    plugin_by_name = dict(zip(name_by_index, plugin_by_index))
    default_plugins = [('UVC_Manager', {}), ('Log_Display', {}),
                       ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}),
                       ('Screen_Marker_Calibration', {}), ('Recorder', {}),
                       ('Pupil_Remote', {}), ('Fixation_Detector_3D', {})]

    # Callback functions
    def on_resize(window, w, h):
        if is_window_visible(window):
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            graph.adjust_size(w, h)
            adjust_gl_view(w, h)
            for p in g_pool.plugins:
                p.on_window_resize(window, w, h)

    def on_iconify(window, iconified):
        g_pool.iconified = iconified

    def on_key(window, key, scancode, action, mods):
        g_pool.gui.update_key(key, scancode, action, mods)

    def on_char(window, char):
        g_pool.gui.update_char(char)

    def on_button(window, button, action, mods):
        g_pool.gui.update_button(button, action, mods)
        pos = glfw.glfwGetCursorPos(window)
        pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
        pos = denormalize(
            pos,
            (frame.img.shape[1], frame.img.shape[0]))  # Position in img pixels
        for p in g_pool.plugins:
            p.on_click(pos, button, action)

    def on_pos(window, x, y):
        hdpi_factor = float(
            glfw.glfwGetFramebufferSize(window)[0] /
            glfw.glfwGetWindowSize(window)[0])
        x, y = x * hdpi_factor, y * hdpi_factor
        g_pool.gui.update_mouse(x, y)

    def on_scroll(window, x, y):
        g_pool.gui.update_scroll(x, y * scroll_factor)

    tick = delta_t()

    def get_dt():
        return next(tick)

    g_pool.on_frame_size_change = lambda new_size: None

    # load session persistent settings
    session_settings = Persistent_Dict(
        os.path.join(g_pool.user_dir, 'user_settings_world'))
    if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
        logger.info(
            "Session setting are from older version of this app. I will not use those."
        )
        session_settings.clear()

    # Initialize capture
    default_settings = {
        'source_class_name':
        'UVC_Source',
        'preferred_names': [
            "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525",
            "C525", "C615", "C920", "C930e"
        ],
        'frame_size': (1280, 720),
        'frame_rate':
        30
    }
    settings = session_settings.get('capture_settings', default_settings)
    try:
        cap = source_by_name[settings['source_class_name']](g_pool, **settings)
    except (KeyError, InitialisationError) as e:
        if isinstance(e, KeyError):
            logger.warning(
                'Incompatible capture setting encountered. Falling back to fake source.'
            )
        cap = Fake_Source(g_pool, **settings)

    g_pool.iconified = False
    g_pool.detection_mapping_mode = session_settings.get(
        'detection_mapping_mode', '3d')
    g_pool.active_calibration_plugin = None
    g_pool.active_gaze_mapping_plugin = None
    g_pool.capture = cap
    g_pool.capture_manager = None

    audio.audio_mode = session_settings.get('audio_mode',
                                            audio.default_audio_mode)

    def open_plugin(plugin):
        if plugin == "Select to load":
            return
        g_pool.plugins.add(plugin)

    def set_scale(new_scale):
        g_pool.gui.scale = new_scale
        g_pool.gui.collect_menus()

    def launch_eye_process(eye_id, delay=0):
        n = {
            'subject': 'eye_process.should_start.%s' % eye_id,
            'eye_id': eye_id,
            'delay': delay
        }
        ipc_pub.notify(n)

    def stop_eye_process(eye_id):
        n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id}
        ipc_pub.notify(n)

    def start_stop_eye(eye_id, make_alive):
        if make_alive:
            launch_eye_process(eye_id)
        else:
            stop_eye_process(eye_id)

    def set_detection_mapping_mode(new_mode):
        n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
        ipc_pub.notify(n)

    def handle_notifications(n):
        subject = n['subject']
        if subject == 'set_detection_mapping_mode':
            if n['mode'] == '2d':
                if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name:
                    logger.warning(
                        "The gaze mapper is not supported in 2d mode. Please recalibrate."
                    )
                    g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
            g_pool.detection_mapping_mode = n['mode']
        elif subject == 'start_plugin':
            g_pool.plugins.add(plugin_by_name[n['name']],
                               args=n.get('args', {}))
        elif subject == 'eye_process.started':
            n = {
                'subject': 'set_detection_mapping_mode',
                'mode': g_pool.detection_mapping_mode
            }
            ipc_pub.notify(n)
        elif subject.startswith('meta.should_doc'):
            ipc_pub.notify({
                'subject': 'meta.doc',
                'actor': g_pool.app,
                'doc': world.__doc__
            })
            for p in g_pool.plugins:
                if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify:
                    ipc_pub.notify({
                        'subject': 'meta.doc',
                        'actor': p.class_name,
                        'doc': p.on_notify.__doc__
                    })

    #window and gl setup
    glfw.glfwInit()
    width, height = session_settings.get('window_size', cap.frame_size)
    main_window = glfw.glfwCreateWindow(width, height, "World")
    window_pos = session_settings.get('window_position',
                                      window_position_default)
    glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
    glfw.glfwMakeContextCurrent(main_window)
    cygl.utils.init()
    g_pool.main_window = main_window

    #setup GUI
    g_pool.gui = ui.UI()
    g_pool.gui.scale = session_settings.get('gui_scale', 1)
    g_pool.sidebar = ui.Scrolling_Menu("Settings",
                                       pos=(-350, 0),
                                       size=(0, 0),
                                       header_pos='left')
    general_settings = ui.Growing_Menu('General')
    general_settings.append(
        ui.Slider('scale',
                  g_pool.gui,
                  setter=set_scale,
                  step=.05,
                  min=1.,
                  max=2.5,
                  label='Interface size'))
    general_settings.append(
        ui.Button(
            'Reset window size', lambda: glfw.glfwSetWindowSize(
                main_window, frame.width, frame.height)))
    general_settings.append(
        ui.Selector('audio_mode', audio, selection=audio.audio_modes))
    general_settings.append(
        ui.Selector('detection_mapping_mode',
                    g_pool,
                    label='detection & mapping mode',
                    setter=set_detection_mapping_mode,
                    selection=['2d', '3d']))
    general_settings.append(
        ui.Switch('eye0_process',
                  label='Detect eye 0',
                  setter=lambda alive: start_stop_eye(0, alive),
                  getter=lambda: eyes_are_alive[0].value))
    general_settings.append(
        ui.Switch('eye1_process',
                  label='Detect eye 1',
                  setter=lambda alive: start_stop_eye(1, alive),
                  getter=lambda: eyes_are_alive[1].value))

    selector_label = "Select to load"
    labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins]
    user_launchable_plugins.insert(0, selector_label)
    labels.insert(0, selector_label)
    general_settings.append(
        ui.Selector('Open plugin',
                    selection=user_launchable_plugins,
                    labels=labels,
                    setter=open_plugin,
                    getter=lambda: selector_label))

    general_settings.append(
        ui.Info_Text('Capture Version: %s' % g_pool.version))

    g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))

    g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
    g_pool.capture.init_gui()

    g_pool.calibration_menu = ui.Growing_Menu('Calibration')
    g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')

    g_pool.sidebar.append(general_settings)
    g_pool.sidebar.append(g_pool.capture_selector_menu)
    g_pool.sidebar.append(g_pool.capture_source_menu)
    g_pool.sidebar.append(g_pool.calibration_menu)

    g_pool.gui.append(g_pool.sidebar)
    g_pool.gui.append(g_pool.quickbar)

    #plugins that are loaded based on user settings from previous session
    g_pool.plugins = Plugin_List(
        g_pool, plugin_by_name,
        session_settings.get('loaded_plugins', default_plugins))

    #We add the calibration menu selector, after a calibration has been added:
    g_pool.calibration_menu.insert(
        0,
        ui.Selector(
            'active_calibration_plugin',
            getter=lambda: g_pool.active_calibration_plugin.__class__,
            selection=calibration_plugins,
            labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
            setter=open_plugin,
            label='Method'))

    #We add the capture selection menu, after a manager has been added:
    g_pool.capture_selector_menu.insert(
        0,
        ui.Selector('capture_manager',
                    setter=open_plugin,
                    getter=lambda: g_pool.capture_manager.__class__,
                    selection=manager_classes,
                    labels=[b.gui_name for b in manager_classes],
                    label='Manager'))

    # Register callbacks main_window
    glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
    glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
    glfw.glfwSetKeyCallback(main_window, on_key)
    glfw.glfwSetCharCallback(main_window, on_char)
    glfw.glfwSetMouseButtonCallback(main_window, on_button)
    glfw.glfwSetCursorPosCallback(main_window, on_pos)
    glfw.glfwSetScrollCallback(main_window, on_scroll)

    # gl_state settings
    basic_gl_setup()
    g_pool.image_tex = Named_Texture()
    # refresh speed settings
    glfw.glfwSwapInterval(0)

    #trigger setup of window and gl sizes
    on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

    #now the we have  aproper window we can load the last gui configuration
    g_pool.gui.configuration = session_settings.get('ui_config', {})

    #set up performace graphs:
    pid = os.getpid()
    ps = psutil.Process(pid)
    ts = g_pool.get_timestamp()

    cpu_graph = graph.Bar_Graph()
    cpu_graph.pos = (20, 130)
    cpu_graph.update_fn = ps.cpu_percent
    cpu_graph.update_rate = 5
    cpu_graph.label = 'CPU %0.1f'

    fps_graph = graph.Bar_Graph()
    fps_graph.pos = (140, 130)
    fps_graph.update_rate = 5
    fps_graph.label = "%0.0f FPS"

    pupil0_graph = graph.Bar_Graph(max_val=1.0)
    pupil0_graph.pos = (260, 130)
    pupil0_graph.update_rate = 5
    pupil0_graph.label = "id0 conf: %0.2f"
    pupil1_graph = graph.Bar_Graph(max_val=1.0)
    pupil1_graph.pos = (380, 130)
    pupil1_graph.update_rate = 5
    pupil1_graph.label = "id1 conf: %0.2f"
    pupil_graphs = pupil0_graph, pupil1_graph

    if session_settings.get('eye1_process_alive', False):
        launch_eye_process(1, delay=0.6)
    if session_settings.get('eye0_process_alive', True):
        launch_eye_process(0, delay=0.3)

    ipc_pub.notify({'subject': 'world_process.started'})
    logger.warning('Process started.')

    # Event loop
    while not glfw.glfwWindowShouldClose(main_window):

        # fetch newest notifications
        new_notifications = []
        while notify_sub.new_data:
            t, n = notify_sub.recv()
            new_notifications.append(n)

        # notify each plugin if there are new notifications:
        for n in new_notifications:
            handle_notifications(n)
            g_pool.capture.on_notify(n)
            for p in g_pool.plugins:
                p.on_notify(n)

        # Get an image from the grabber
        try:
            frame = g_pool.capture.get_frame()
        except StreamError as e:
            prev_settings = g_pool.capture.settings
            g_pool.capture.deinit_gui()
            g_pool.capture.cleanup()
            g_pool.capture = None
            prev_settings[
                'info_text'] = "'%s' disconnected." % prev_settings['name']
            g_pool.capture = Fake_Source(g_pool, **prev_settings)
            g_pool.capture.init_gui()
            ipc_pub.notify({'subject': 'recording.should_stop'})
            logger.error("Error getting frame. Falling back to Fake source.")
            logger.debug("Caught error: %s" % e)
            sleep(.2)
            continue
        except EndofVideoFileError:
            logger.warning("Video file is done. Rewinding")
            g_pool.capture.seek_to_frame(0)
            continue

        #update performace graphs
        t = frame.timestamp
        dt, ts = t - ts, t
        try:
            fps_graph.add(1. / dt)
        except ZeroDivisionError:
            pass
        cpu_graph.update()

        #a dictionary that allows plugins to post and read events
        events = {}
        #report time between now and the last loop interation
        events['dt'] = get_dt()

        recent_pupil_data = []
        recent_gaze_data = []

        while pupil_sub.new_data:
            t, p = pupil_sub.recv()
            pupil_graphs[p['id']].add(p['confidence'])
            recent_pupil_data.append(p)
            new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p)
            for g in new_gaze_data:
                gaze_pub.send('gaze', g)
            recent_gaze_data += new_gaze_data

        events['pupil_positions'] = recent_pupil_data
        events['gaze_positions'] = recent_gaze_data

        # allow each Plugin to do its work.
        for p in g_pool.plugins:
            p.update(frame, events)

        #check if a plugin need to be destroyed
        g_pool.plugins.clean()

        #send new events to ipc:
        del events['pupil_positions']  #already on the wire
        del events['gaze_positions']  #send earlier in this loop
        del events['dt']  #no need to send this
        for topic, data in events.iteritems():
            assert (isinstance(data, (list, tuple)))
            for d in data:
                ipc_pub.send(topic, d)

        # render camera image
        glfw.glfwMakeContextCurrent(main_window)
        if is_window_visible(main_window):
            g_pool.image_tex.update_from_frame(frame)
            glFlush()
        make_coord_system_norm_based()
        g_pool.image_tex.draw()
        make_coord_system_pixel_based((frame.height, frame.width, 3))
        # render visual feedback from loaded plugins

        if is_window_visible(main_window):
            g_pool.capture.gl_display()
            for p in g_pool.plugins:
                p.gl_display()

            graph.push_view()
            fps_graph.draw()
            cpu_graph.draw()
            pupil0_graph.draw()
            pupil1_graph.draw()
            graph.pop_view()
            g_pool.gui.update()
            glfw.glfwSwapBuffers(main_window)
        glfw.glfwPollEvents()

    glfw.glfwRestoreWindow(main_window)  #need to do this for windows os
    session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
    session_settings['gui_scale'] = g_pool.gui.scale
    session_settings['ui_config'] = g_pool.gui.configuration
    session_settings['capture_settings'] = g_pool.capture.settings
    session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
    session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
    session_settings['version'] = g_pool.version
    session_settings['eye0_process_alive'] = eyes_are_alive[0].value
    session_settings['eye1_process_alive'] = eyes_are_alive[1].value
    session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
    session_settings['audio_mode'] = audio.audio_mode
    session_settings.close()

    # de-init all running plugins
    for p in g_pool.plugins:
        p.alive = False
    g_pool.plugins.clean()
    g_pool.gui.terminate()
    glfw.glfwDestroyWindow(main_window)
    glfw.glfwTerminate()

    g_pool.capture.deinit_gui()
    g_pool.capture.cleanup()

    #shut down eye processes:
    stop_eye_process(0)
    stop_eye_process(1)

    logger.info("Process shutting down.")
    ipc_pub.notify({'subject': 'world_process.stopped'})

    #shut down launcher
    n = {'subject': 'launcher_process.should_stop'}
    ipc_pub.notify(n)
    zmq_ctx.destroy()
Ejemplo n.º 30
0
 def gl_display_in_window(self):
     try:
         active_window = glfwGetCurrentContext()
         if glfwWindowShouldClose(self._window):
             self.close_window()
             return
             
         # make plugin window current context and clear the screen
         glfwMakeContextCurrent(self._window)
         clear_gl_screen()
         
         gl.glMatrixMode(gl.GL_PROJECTION)
         gl.glLoadIdentity()
         p_window_size = glfwGetFramebufferSize(self._window)
         gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
         # Switch back to Model View Matrix
         gl.glMatrixMode(gl.GL_MODELVIEW)
         gl.glLoadIdentity()
         
         # draw markers using the previously generated ndarrays and Named_Texture objects 
         # (see https://github.com/pupil-labs/pyglui/blob/master/pyglui/cygl/utils.pyx for details)
         m1 = Named_Texture()
         m1.update_from_ndarray(self.marker1)
         m1.draw(True, ((10.0, self.m_size), (self.m_size, self.m_size), (self.m_size, 10.0), (10.0, 10.0)), 10.0)
         
         m2 = Named_Texture()
         m2.update_from_ndarray(self.marker2)
         m2.draw(True, ((p_window_size[0]-self.m_size, self.m_size), (p_window_size[0]-10.0, self.m_size), (p_window_size[0]-10.0, 10.0), (p_window_size[0]-self.m_size, 10.0)), 10.0)
         
         m3 = Named_Texture()
         m3.update_from_ndarray(self.marker3)
         m3.draw(True, ((10.0, p_window_size[1]-10.0), (self.m_size, p_window_size[1]-10.0), (self.m_size, p_window_size[1]-self.m_size), (10.0, p_window_size[1]-self.m_size)), 10.0)
         
         m4 = Named_Texture()
         m4.update_from_ndarray(self.marker4)
         m4.draw(True, ((p_window_size[0]-self.m_size, p_window_size[1]-10.0), (p_window_size[0]-10.0, p_window_size[1]-10.0), (p_window_size[0]-10.0, p_window_size[1]-self.m_size), (p_window_size[0]-self.m_size, p_window_size[1]-self.m_size)), 10.0)
         
         # swap buffer
         glfwSwapBuffers(self._window)
         glfwMakeContextCurrent(active_window)
     except:
         logger.error("Unexpected error: {}".format(sys.exc_info()))