Ejemplo n.º 1
0
def remove_outliers(limbuses):

    limbuses_to_return = [limbuses[0], limbuses[1]]

    for i, limbus in enumerate(limbuses):

        # Remove limbuses that point widly out of frame
        if limbus is None:
            limbuses_to_return[i] = None
            continue

        gaze_pts_mm_x, gaze_pts_mm_y = gaze_geometry.get_gaze_point_mm(limbus)
        if gaze_pts_mm_x < -screen_w_mm_n7 or gaze_pts_mm_x > screen_w_mm_n7:
            limbuses_to_return[i] = None
            continue
        elif gaze_pts_mm_y < (-screen_h_mm_n7 * 3 / 2) or (gaze_pts_mm_y > 0):
            limbuses_to_return[i] = None
            continue
        else:
            limbuses_to_return[i] = limbus

    # Remove limbuses that are too far from eachother (Pupilary Distance)
    if limbuses_to_return[0] is not None and limbuses_to_return[1] is not None:

        l1x, l1y, l1z = limbuses_to_return[0].center_mm
        l2x, l2y, l2z = limbuses_to_return[1].center_mm
        limb_dist = math.sqrt((l1x - l2x)**2 + (l1y - l2y)**2 + (l1z - l2z)**2)

        last_x, last_y, last_z = __last_pos

        if limb_dist > 80:
            dist_1 = (last_x - l1x)**2 + (last_y - l1y)**2 + (last_z - l1z)**2
            dist_2 = (last_x - l2x)**2 + (last_y - l2y)**2 + (last_z - l2z)**2
            if dist_1 < dist_2:
                limbuses_to_return[1] = None
            else:
                limbuses_to_return[0] = None

        else:
            __last_pos[0], __last_pos[1], __last_pos[2] = (l1x + l2x) / 2, (
                l1y + l2y) / 2, (l1z + l2z) / 2

    return limbuses_to_return
Ejemplo n.º 2
0
    def update_vis(self, limbuses, smoothed_gaze_pt_mm=None):
        
        self.handle_keyboard()
        
        for i, limbus in enumerate(limbuses):
        
            # Change colors, continue to next eye if limbus is none (not found)
            limb_color = cvt_col(debug_colors[3 if limbus is None else i])
            self.limb_circles[i].color = limb_color
            self.limb_normals[i].color = limb_color
            if limbus is None: continue
        
            # Show 3D limbus circle
            self.limb_circles[i].pos = cvt_pt(*limbus.center_mm)
            self.limb_circles[i].axis = cvt_pt(*limbus.normal)
            self.limb_circles[i].length = 0.1
            
            # Move eye-sphere
            self.eye_spheres[i].pos = self.limb_circles[i].pos - self.limb_circles[i].axis * 100
            
            # Show intersection with normal on screen plane
            gaze_pt_x_mm, gaze_pt_y_mm = gaze_geometry.get_gaze_point_mm(limbus)
            
            # Either show long or short limb normal
            self.limb_normals[i].pos = [cvt_pt(*limbus.center_mm),
                                        cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0) if self.vis_mode == 0 else
                                        (self.limb_circles[i].pos + self.limb_circles[i].axis * 200)]
            
            # Show raw gaze points and trail
            self.raw_gaze_pts[i].pos = cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0.01)
            self.raw_gaze_pts[i].trail_object.color = self.raw_gaze_pts_trail_colors[i]
            
            # Label eye position
            # self.limb_labels[i].pos = self.limb_circles[i].pos
            # self.limb_labels[i].text = '%0.2d, %0.2d, %0.2d' % limbus.center_mm

        # Handle smoothed gaze pt and save evaluation data
        if smoothed_gaze_pt_mm is not None:
            sm_gaze_pt_x, sm_gaze_pt_y = smoothed_gaze_pt_mm
            self.smoothed_gaze_pt.pos = cvt_pt(sm_gaze_pt_x, sm_gaze_pt_y, 0)
            self.smoothed_gaze_pt.trail_object.color = self.smoothed_gaze_pt_trail_colors
            
            limb_mid_pt = (self.limb_circles[0].pos + self.limb_circles[1].pos) / 2
            end_pt = self.smoothed_gaze_pt.pos if self.vis_mode == 1 else limb_mid_pt
            smoothed_vector_pts = [(1 - a) * limb_mid_pt + a * end_pt for a in np.arange(0, 1.05, 0.05)]
            self.smoothed_gaze_vector.pos = smoothed_vector_pts
            self.smoothed_gaze_vector.color = self.smoothed_gaze_pt_trail_colors
            
            # ---------------------------------
            # SAVE EVALUATION DATA
            # ---------------------------------
            if self.eval_file is not None:
                
                # Only record gaze-data after short cooldown
                if self.marker_switch_cooldown <= 0:
                
                    correct_pos = self.markers[self.active_marker_vpy_id].pos
                    gaze_pos = self.smoothed_gaze_pt.pos
                    dist_offset = mag(correct_pos - gaze_pos)
                    x_offset = abs(correct_pos.x - gaze_pos.x)
                    correct_vec = correct_pos - limb_mid_pt
                    gaze_vec = gaze_pos - limb_mid_pt
                    angle_offset = math.degrees(correct_vec.diff_angle(gaze_vec))
                    
                    # marker x,y, gaze x,y, dist to eye-pair, dist_err, dist_err_x, angle_err
                    line_to_write = ','.join([str(x) for x in
                                              [self.active_marker_ind,
                                               ("%.4f" % correct_pos.x), ("%.4f" % correct_pos.y),
                                               ("%.4f" % gaze_pos.x), ("%.4f" % gaze_pos.y),
                                               ("%.4f" % mag(limb_mid_pt)),
                                               ("%.4f" % dist_offset), ("%.4f" % x_offset), ("%.4f" % angle_offset)]])
                    self.eval_file.write(line_to_write + '\n')
                    
                else:
                    self.marker_switch_cooldown -= 1
Ejemplo n.º 3
0
    def get_gaze_from_frame(self, frame):

        frame = cv2.undistort(frame, cam_mat_n7, dist_coefs_n7)

        frame_pyr = image_utils.make_gauss_pyr(frame, 4)
        full_frame = frame_pyr[1].copy()
        half_frame = frame_pyr[2].copy()

        limbuses = [None, None]
        gaze_pts_mm = [None, None]
        gaze_pts_px = [None, None]

        try:
            sub_img_cx0, sub_img_cy0 = None, None
            eye_r_roi, eye_l_roi = eye_extractor.get_eye_rois(
                frame_pyr, 4, debug=self.debug, device=self.device)

            for i, eye_roi in enumerate([eye_r_roi, eye_l_roi]):

                try:
                    if eye_roi.img is None: break

                    # Gives unique winnames for each ROI
                    debug_index = ((i + 1) if self.debug else False)

                    eye_roi.img = self.pre_proc.erase_specular(
                        eye_roi.img, debug=debug_index)

                    pupil_x0, pupil_y0 = eye_center_locator_combined.find_pupil(
                        eye_roi.img,
                        fast_width_grads=25.0,
                        fast_width_iso=80.0,
                        weight_grads=0.8,
                        weight_iso=0.2,
                        debug_index=debug_index)
                    eye_roi.refine_pupil((pupil_x0, pupil_y0), full_frame)
                    roi_x0, roi_y0, roi_w, roi_h = eye_roi.roi_x0, eye_roi.roi_y0, eye_roi.roi_w, eye_roi.roi_h

                    u_eyelid, l_eyelid = find_eyelids(eye_roi.img, debug_index)

                    pts_found = get_limb_pts(eye_img=eye_roi.img,
                                             phi=20,
                                             angle_step=1,
                                             debug_index=debug_index)
                    pts_found = eyelid_locator.filter_limbus_pts(
                        u_eyelid, l_eyelid, pts_found)

                    ellipse = ransac_ellipse.ransac_ellipse_fit(
                        points=pts_found,
                        bgr_img=eye_roi.img,
                        roi_pos=(roi_x0, roi_y0),
                        ransac_iters_max=5,
                        refine_iters_max=3,
                        max_err=1,
                        debug=False)

                    # Shift 2D limbus ellipse and points to account for eye ROI coords
                    (ell_x0, ell_y0), (ell_w,
                                       ell_h), angle = ellipse.rotated_rect
                    new_rotated_rect = (roi_x0 + ell_x0,
                                        roi_y0 + ell_y0), (ell_w, ell_h), angle
                    ellipse = Ellipse(new_rotated_rect)
                    pts_found_to_draw = [(px + roi_x0, py + roi_y0)
                                         for (px, py) in pts_found]

                    # Correct coords when extracting eye for half-frame
                    (sub_img_cx0, sub_img_cy0) = (roi_x0 + ell_x0,
                                                  roi_y0 + ell_y0)

                    # Ignore incorrect limbus
                    limbus = gaze_geometry.ellipse_to_limbuses_persp_geom(
                        ellipse, self.device)
                    limbuses[i] = limbus

                    # Draw eye features onto debug image
                    draw_utils.draw_limbus(full_frame,
                                           limbus,
                                           color=debug_colors[i],
                                           scale=1)
                    draw_utils.draw_points(full_frame,
                                           pts_found_to_draw,
                                           color=debug_colors[i],
                                           width=1,
                                           thickness=2)
                    draw_utils.draw_normal(full_frame,
                                           limbus,
                                           self.device,
                                           color=debug_colors[i],
                                           scale=1)
                    draw_utils.draw_normal(half_frame,
                                           limbus,
                                           self.device,
                                           color=debug_colors[i],
                                           scale=0.5,
                                           arrow_len_mm=20)
                    eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 +
                                                         eye_roi.roi_h),
                                         eye_roi.roi_x0:(eye_roi.roi_x0 +
                                                         eye_roi.roi_w)]
                    draw_utils.draw_eyelids(u_eyelid, l_eyelid, eye_img)

                except ransac_ellipse.NoEllipseFound:
                    if self.debug: print 'No Ellipse Found'
                    cv2.rectangle(full_frame, (roi_x0, roi_y0),
                                  (roi_x0 + roi_w, roi_y0 + roi_h),
                                  (0, 0, 255),
                                  thickness=4)

                except ransac_ellipse.CoverageTooLow as e:
                    if self.debug:
                        print 'Ellipse Coverage Too Low : %s' % e.msg
                    cv2.rectangle(full_frame, (roi_x0, roi_y0),
                                  (roi_x0 + roi_w, roi_y0 + roi_h),
                                  (0, 0, 255),
                                  thickness=4)

                finally:

                    # Extract only eye_roi block after other drawing methods
                    if sub_img_cx0 is not None:
                        eye_img = full_frame[sub_img_cy0 - 60:sub_img_cy0 + 60,
                                             sub_img_cx0 - 60:sub_img_cx0 + 60]
                    else:
                        eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 +
                                                             eye_roi.roi_h),
                                             eye_roi.roi_x0:(eye_roi.roi_x0 +
                                                             eye_roi.roi_w)]

                    # Transfer eye_img block to section of half_frame
                    half_frame[half_frame.shape[0] -
                               eye_img.shape[0]:half_frame.shape[0],
                               (half_frame.shape[1] - eye_img.shape[1]) *
                               i:half_frame.shape[1] if i else eye_img.
                               shape[1]] = eye_img

        except eye_extractor.NoEyesFound as e:
            if self.debug: print 'No Eyes Found: %s' % e.msg

        # Remove any extreme outliers
        limbuses = limbus_outlier_removal.remove_outliers(limbuses)

        # Get gaze points
        for i, limbus in enumerate(limbuses):
            if limbus is None: continue
            gaze_pts_mm[i] = gaze_geometry.get_gaze_point_mm(limbus)
            gaze_pts_px[i] = gaze_geometry.convert_gaze_pt_mm_to_px(
                gaze_pts_mm[i], self.device)

        smoothed_gaze_pt_mm = self.smoother.smooth_gaze(gaze_pts_mm)
        smoothed_gaze_pt_px = gaze_geometry.convert_gaze_pt_mm_to_px(
            smoothed_gaze_pt_mm, self.device)

        # Visualize in 2D and 3D
        cv2.imshow('gaze system', half_frame)
        self.visualizer3d.update_vis(limbuses, smoothed_gaze_pt_mm)

        # If recording, take a screenshot of vpython and add to vid. capture
        if self.recording:
            vis_screen = self.visualizer3d.take_screenshot()
            stacked_imgs = image_utils.stack_imgs_horizontal(
                [vis_screen, half_frame])
            self.vid_writer.write(stacked_imgs)

        return smoothed_gaze_pt_px
Ejemplo n.º 4
0
 def get_gaze_from_frame(self, frame):
     
     frame = cv2.undistort(frame, cam_mat_n7, dist_coefs_n7)
     
     frame_pyr = image_utils.make_gauss_pyr(frame, 4)
     full_frame = frame_pyr[1].copy()
     half_frame = frame_pyr[2].copy()
     
     limbuses = [None, None]
     gaze_pts_mm = [None, None]
     gaze_pts_px = [None, None]
     
     try:
         sub_img_cx0, sub_img_cy0 = None, None
         eye_r_roi, eye_l_roi = eye_extractor.get_eye_rois(frame_pyr, 4, debug=self.debug, device=self.device)
         
         for i, eye_roi in enumerate([eye_r_roi, eye_l_roi]):
             
             try:
                 if eye_roi.img is None: break
                 
                 # Gives unique winnames for each ROI
                 debug_index = ((i + 1) if self.debug else False)  
         
                 eye_roi.img = self.pre_proc.erase_specular(eye_roi.img, debug=debug_index)
         
                 pupil_x0, pupil_y0 = eye_center_locator_combined.find_pupil(eye_roi.img,
                                                                             fast_width_grads=25.0,
                                                                             fast_width_iso=80.0,
                                                                             weight_grads=0.8,
                                                                             weight_iso=0.2,
                                                                             debug_index=debug_index)
                 eye_roi.refine_pupil((pupil_x0, pupil_y0), full_frame)
                 roi_x0, roi_y0, roi_w, roi_h = eye_roi.roi_x0, eye_roi.roi_y0, eye_roi.roi_w, eye_roi.roi_h
                 
                 u_eyelid, l_eyelid = find_eyelids(eye_roi.img, debug_index)
                 
                 pts_found = get_limb_pts(eye_img=eye_roi.img,
                                          phi=20,
                                          angle_step=1,
                                          debug_index=debug_index)
                 pts_found = eyelid_locator.filter_limbus_pts(u_eyelid, l_eyelid, pts_found)
                 
                 ellipse = ransac_ellipse.ransac_ellipse_fit(points=pts_found,
                                                             bgr_img=eye_roi.img,
                                                             roi_pos=(roi_x0, roi_y0),
                                                             ransac_iters_max=5,
                                                             refine_iters_max=3,
                                                             max_err=1,
                                                             debug=False)
                 
                 # Shift 2D limbus ellipse and points to account for eye ROI coords
                 (ell_x0, ell_y0), (ell_w, ell_h), angle = ellipse.rotated_rect               
                 new_rotated_rect = (roi_x0 + ell_x0, roi_y0 + ell_y0), (ell_w, ell_h), angle
                 ellipse = Ellipse(new_rotated_rect)                                                                                
                 pts_found_to_draw = [(px + roi_x0, py + roi_y0) for (px, py) in pts_found]
                 
                 # Correct coords when extracting eye for half-frame
                 (sub_img_cx0, sub_img_cy0) = (roi_x0 + ell_x0, roi_y0 + ell_y0)
                 
                 # Ignore incorrect limbus
                 limbus = gaze_geometry.ellipse_to_limbuses_persp_geom(ellipse, self.device)
                 limbuses[i] = limbus
                 
                 # Draw eye features onto debug image
                 draw_utils.draw_limbus(full_frame, limbus, color=debug_colors[i], scale=1)
                 draw_utils.draw_points(full_frame, pts_found_to_draw, color=debug_colors[i], width=1, thickness=2)
                 draw_utils.draw_normal(full_frame, limbus, self.device, color=debug_colors[i], scale=1)
                 draw_utils.draw_normal(half_frame, limbus, self.device, color=debug_colors[i], scale=0.5, arrow_len_mm=20)
                 eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h),
                                      eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)]
                 draw_utils.draw_eyelids(u_eyelid, l_eyelid, eye_img)
                 
             except ransac_ellipse.NoEllipseFound:
                 if self.debug: print 'No Ellipse Found'
                 cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4)
                 
             except ransac_ellipse.CoverageTooLow as e:
                 if self.debug: print 'Ellipse Coverage Too Low : %s' % e.msg
                 cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4)
                 
             finally:
                 
                 # Extract only eye_roi block after other drawing methods
                 if sub_img_cx0 is not None: 
                     eye_img = full_frame[sub_img_cy0 - 60:sub_img_cy0 + 60,
                                          sub_img_cx0 - 60:sub_img_cx0 + 60]
                 else:
                     eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h),
                                          eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)]
                 
                 # Transfer eye_img block to section of half_frame
                 half_frame[half_frame.shape[0] - eye_img.shape[0]:half_frame.shape[0],
                            (half_frame.shape[1] - eye_img.shape[1]) * i: half_frame.shape[1] if i else eye_img.shape[1]] = eye_img
                            
     except eye_extractor.NoEyesFound as e:
         if self.debug: print 'No Eyes Found: %s' % e.msg
         
     # Remove any extreme outliers
     limbuses = limbus_outlier_removal.remove_outliers(limbuses)
     
     # Get gaze points
     for i, limbus in enumerate(limbuses):
         if limbus is None: continue
         gaze_pts_mm[i] = gaze_geometry.get_gaze_point_mm(limbus)
         gaze_pts_px[i] = gaze_geometry.convert_gaze_pt_mm_to_px(gaze_pts_mm[i], self.device)
     
     smoothed_gaze_pt_mm = self.smoother.smooth_gaze(gaze_pts_mm)
     smoothed_gaze_pt_px = gaze_geometry.convert_gaze_pt_mm_to_px(smoothed_gaze_pt_mm, self.device)
     
     # Visualize in 2D and 3D
     cv2.imshow('gaze system', half_frame)
     self.visualizer3d.update_vis(limbuses, smoothed_gaze_pt_mm)
     
     # If recording, take a screenshot of vpython and add to vid. capture
     if self.recording:
         vis_screen = self.visualizer3d.take_screenshot()
         stacked_imgs = image_utils.stack_imgs_horizontal([vis_screen, half_frame])
         self.vid_writer.write(stacked_imgs)
         
     return smoothed_gaze_pt_px
Ejemplo n.º 5
0
    def update_vis(self, limbuses, smoothed_gaze_pt_mm=None):

        self.handle_keyboard()

        for i, limbus in enumerate(limbuses):

            # Change colors, continue to next eye if limbus is none (not found)
            limb_color = cvt_col(debug_colors[3 if limbus is None else i])
            self.limb_circles[i].color = limb_color
            self.limb_normals[i].color = limb_color
            if limbus is None: continue

            # Show 3D limbus circle
            self.limb_circles[i].pos = cvt_pt(*limbus.center_mm)
            self.limb_circles[i].axis = cvt_pt(*limbus.normal)
            self.limb_circles[i].length = 0.1

            # Move eye-sphere
            self.eye_spheres[i].pos = self.limb_circles[
                i].pos - self.limb_circles[i].axis * 100

            # Show intersection with normal on screen plane
            gaze_pt_x_mm, gaze_pt_y_mm = gaze_geometry.get_gaze_point_mm(
                limbus)

            # Either show long or short limb normal
            self.limb_normals[i].pos = [
                cvt_pt(*limbus.center_mm),
                cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0)
                if self.vis_mode == 0 else
                (self.limb_circles[i].pos + self.limb_circles[i].axis * 200)
            ]

            # Show raw gaze points and trail
            self.raw_gaze_pts[i].pos = cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0.01)
            self.raw_gaze_pts[
                i].trail_object.color = self.raw_gaze_pts_trail_colors[i]

            # Label eye position
            # self.limb_labels[i].pos = self.limb_circles[i].pos
            # self.limb_labels[i].text = '%0.2d, %0.2d, %0.2d' % limbus.center_mm

        # Handle smoothed gaze pt and save evaluation data
        if smoothed_gaze_pt_mm is not None:
            sm_gaze_pt_x, sm_gaze_pt_y = smoothed_gaze_pt_mm
            self.smoothed_gaze_pt.pos = cvt_pt(sm_gaze_pt_x, sm_gaze_pt_y, 0)
            self.smoothed_gaze_pt.trail_object.color = self.smoothed_gaze_pt_trail_colors

            limb_mid_pt = (self.limb_circles[0].pos +
                           self.limb_circles[1].pos) / 2
            end_pt = self.smoothed_gaze_pt.pos if self.vis_mode == 1 else limb_mid_pt
            smoothed_vector_pts = [(1 - a) * limb_mid_pt + a * end_pt
                                   for a in np.arange(0, 1.05, 0.05)]
            self.smoothed_gaze_vector.pos = smoothed_vector_pts
            self.smoothed_gaze_vector.color = self.smoothed_gaze_pt_trail_colors

            # ---------------------------------
            # SAVE EVALUATION DATA
            # ---------------------------------
            if self.eval_file is not None:

                # Only record gaze-data after short cooldown
                if self.marker_switch_cooldown <= 0:

                    correct_pos = self.markers[self.active_marker_vpy_id].pos
                    gaze_pos = self.smoothed_gaze_pt.pos
                    dist_offset = mag(correct_pos - gaze_pos)
                    x_offset = abs(correct_pos.x - gaze_pos.x)
                    correct_vec = correct_pos - limb_mid_pt
                    gaze_vec = gaze_pos - limb_mid_pt
                    angle_offset = math.degrees(
                        correct_vec.diff_angle(gaze_vec))

                    # marker x,y, gaze x,y, dist to eye-pair, dist_err, dist_err_x, angle_err
                    line_to_write = ','.join([
                        str(x) for x in [
                            self.active_marker_ind, ("%.4f" % correct_pos.x),
                            ("%.4f" % correct_pos.y), ("%.4f" % gaze_pos.x),
                            ("%.4f" % gaze_pos.y), ("%.4f" % mag(limb_mid_pt)),
                            ("%.4f" %
                             dist_offset), ("%.4f" %
                                            x_offset), ("%.4f" % angle_offset)
                        ]
                    ])
                    self.eval_file.write(line_to_write + '\n')

                else:
                    self.marker_switch_cooldown -= 1