コード例 #1
0
ファイル: ray_casting.py プロジェクト: zhoushiwei/EyeTab
    def find_limbus_edge_pts(self, eye_roi, debug=False):

        blurred_eye_roi_img = cv2.GaussianBlur(eye_roi.img, (5, 5), 5)

        pupil_x0, pupil_y0 = eye_roi.img.shape[1] / 2, eye_roi.img.shape[0] / 2

        min_limb_r = int(eye_roi.img.shape[0] * self.limb_r_ratios[0])
        max_limb_r = int(eye_roi.img.shape[0] * self.limb_r_ratios[1])

        pts_found = set()
        pts_found = pts_found.union(
            self.cast_rays_spread(bgr_img=blurred_eye_roi_img,
                                  start_pos=(pupil_x0, pupil_y0),
                                  angle_mean=0,
                                  spread=120,
                                  limb_r_range=(min_limb_r, max_limb_r)))
        pts_found = pts_found.union(
            self.cast_rays_spread(bgr_img=blurred_eye_roi_img,
                                  start_pos=(pupil_x0, pupil_y0),
                                  angle_mean=180,
                                  spread=120,
                                  limb_r_range=(min_limb_r, max_limb_r)))

        if debug:
            debug_img = blurred_eye_roi_img.copy()
            cv2.circle(blurred_eye_roi_img,
                       (eye_roi.img.shape[1] / 2, eye_roi.img.shape[0] / 2),
                       min_limb_r, (255, 255, 0))
            cv2.circle(blurred_eye_roi_img,
                       (eye_roi.img.shape[1] / 2, eye_roi.img.shape[0] / 2),
                       max_limb_r, (255, 255, 0))
            draw_cross(debug_img, (pupil_x0, pupil_y0),
                       color=(255, 255, 0),
                       width=6)
            draw_points(debug_img,
                        pts_found, (0, 0, 255),
                        width=1,
                        thickness=2)
            stacked_imgs = np.concatenate(
                [eye_roi.img, blurred_eye_roi_img, debug_img], axis=1)

            if debug == 1:
                self.full_debug_img = stacked_imgs
            elif debug == 2:
                self.full_debug_img = stack_imgs_vertical(
                    [self.full_debug_img, stacked_imgs])
                cv2.imshow(winname, self.full_debug_img)
            elif debug == 3:
                cv2.imshow(winname, stacked_imgs)

        return pts_found
コード例 #2
0
ファイル: ray_casting.py プロジェクト: Cosijopiii/EyeTab
    def find_limbus_edge_pts(self, eye_roi, debug=False):
        
        blurred_eye_roi_img = cv2.GaussianBlur(eye_roi.img, (5, 5), 5)

        pupil_x0, pupil_y0 = eye_roi.img.shape[1] / 2, eye_roi.img.shape[0] / 2
    
        min_limb_r = int(eye_roi.img.shape[0] * self.limb_r_ratios[0])
        max_limb_r = int(eye_roi.img.shape[0] * self.limb_r_ratios[1])
        
        pts_found = set()
        pts_found = pts_found.union(self.cast_rays_spread(bgr_img=blurred_eye_roi_img,
                                                          start_pos=(pupil_x0, pupil_y0),
                                                          angle_mean=0, spread=120,
                                                          limb_r_range=(min_limb_r, max_limb_r)))
        pts_found = pts_found.union(self.cast_rays_spread(bgr_img=blurred_eye_roi_img,
                                                          start_pos=(pupil_x0, pupil_y0),
                                                          angle_mean=180, spread=120,
                                                          limb_r_range=(min_limb_r, max_limb_r)))
                    
        if debug:
            debug_img = blurred_eye_roi_img.copy()
            cv2.circle(blurred_eye_roi_img, (eye_roi.img.shape[1] / 2, eye_roi.img.shape[0] / 2), min_limb_r, (255, 255, 0))
            cv2.circle(blurred_eye_roi_img, (eye_roi.img.shape[1] / 2, eye_roi.img.shape[0] / 2), max_limb_r, (255, 255, 0))
            draw_cross(debug_img, (pupil_x0, pupil_y0), color=(255, 255, 0), width=6)
            draw_points(debug_img, pts_found, (0, 0, 255), width=1, thickness=2)
            stacked_imgs = np.concatenate([eye_roi.img, blurred_eye_roi_img, debug_img], axis=1)
            
            if debug == 1:
                self.full_debug_img = stacked_imgs
            elif debug == 2:
                self.full_debug_img = stack_imgs_vertical([self.full_debug_img, stacked_imgs])
                cv2.imshow(winname, self.full_debug_img)
            elif debug == 3:
                cv2.imshow(winname, stacked_imgs);
        
        return pts_found
コード例 #3
0
def get_limb_pts(eye_img, phi=20, angle_step=1, debug_index=False):

    polar_img_w = 360 / angle_step  # Polar image has one column per angle of interest
    phi_range_1 = ((90 - phi) / angle_step, (90 + phi) / angle_step
                   )  # Ranges of angles to be ignored (too close to lids)
    phi_range_2 = ((270 - phi) / angle_step, (270 + phi) / angle_step)

    eye_img_grey = cv2.cvtColor(eye_img, cv2.COLOR_BGR2GRAY)  # Do BGR-grey
    eye_img_grey = cv2.medianBlur(eye_img_grey, 5)

    # Scale to fixed size image for re-using transform matrix
    scale = eye_img.shape[0] / float(__fixed_width)
    img_fixed_size = cv2.resize(eye_img_grey, (__fixed_width, __fixed_width))

    # Transform image into polar coords and blur
    img_polar = linpolar(img_fixed_size,
                         trans_w=polar_img_w,
                         trans_h=__fixed_width / 2)
    img_polar = cv2.GaussianBlur(img_polar, (5, 5), 0)

    # Take the segment between min & max radii and filter with Gabor kernel
    img_polar_seg = img_polar[__min_limb_r:__max_limb_r, :]
    filter_img = cv2.filter2D(img_polar_seg, -1, __gabor_kern)

    # Black out ignored angles
    filter_img.T[phi_range_1[0]:phi_range_1[1]] = 0
    filter_img.T[phi_range_2[0]:phi_range_2[1]] = 0

    # In polar image, x <-> theta, y <-> magnitude
    pol_ys = np.argmax(filter_img,
                       axis=0)  # Take highest filter response as limbus points
    pol_xs = np.arange(filter_img.shape[1])[pol_ys > 0]
    mags = (pol_ys + __min_limb_r)[pol_ys > 0]
    thts = np.radians(pol_xs * angle_step)

    # Translate each point back into fixed img coords
    xs, ys = cv2.polarToCart(mags.astype(float), thts)
    xs = (
        xs + __fixed_width / 2
    ) * scale  # Shift and scale cart. coords back to original eye-ROI coords
    ys = (ys + __fixed_width / 2) * scale

    # Points returned in form
    #    [[ x1   y1]
    #     [ x2   y2]
    #         ...
    #     [ xn   yn]]
    pts_cart = np.concatenate([xs, ys], axis=1)

    # --------------------- Debug Drawing ---------------------
    if debug_index != False:
        debug_img = eye_img.copy()
        debug_polar = cv2.cvtColor(img_polar, cv2.COLOR_GRAY2BGR)

        cv2.imwrite("polar.jpg", debug_polar)

        cv2.line(debug_polar, (0, __min_limb_r),
                 (img_polar.shape[1], __min_limb_r), (255, 255, 0))
        cv2.line(debug_polar, (0, __max_limb_r),
                 (img_polar.shape[1], __max_limb_r), (255, 255, 0))
        cv2.circle(debug_img, (debug_img.shape[1] / 2, debug_img.shape[0] / 2),
                   int(debug_img.shape[0] * __limb_r_ratios[0]), (255, 255, 0))
        cv2.circle(debug_img, (debug_img.shape[1] / 2, debug_img.shape[0] / 2),
                   int(debug_img.shape[0] * __limb_r_ratios[1]), (255, 255, 0))

        pts_polar = np.squeeze(np.dstack([pol_xs, mags]))
        draw_points(debug_polar, pts_polar, (0, 0, 255), width=1)
        draw_points(debug_img, pts_cart, (0, 0, 255), width=1)

        stacked_imgs_polar = stack_imgs_vertical([debug_polar, filter_img])
        stacked_imgs = stack_imgs_horizontal(
            [debug_img, eye_img_grey, stacked_imgs_polar])

        __debug_imgs[debug_index] = stacked_imgs

        if debug_index == 2:
            full_debug_img = stack_imgs_vertical(
                [__debug_imgs[1], __debug_imgs[2]])
            cv2.imshow(__winname, full_debug_img)
        elif debug_index > 2:
            cv2.imshow(__winname, stacked_imgs)
    # --------------------- Debug Drawing ---------------------

    return pts_cart
コード例 #4
0
ファイル: eyelid_locator.py プロジェクト: zhoushiwei/EyeTab
def find_lower_eyelid(eye_img, debug_index):

    line_y_offset = 0  # Amount to shift eye-lid by after detection

    img_blue = cv2.split(eye_img)[2]
    img_w, img_h = eye_img.shape[:2]

    # Indexes to extract window sub-images
    w_y1, w_y2 = int(img_h * __l_win_rats_h[0]), int(img_h *
                                                     sum(__l_win_rats_h[:2]))
    wl_x1, wl_x2 = int(img_w * __l_win_rats_w_l[0]), int(
        img_w * sum(__l_win_rats_w_l[:2]))
    wr_x1, wr_x2 = int(img_w * __l_win_rats_w_r[0]), int(
        img_w * sum(__l_win_rats_w_r[:2]))

    # Split image into two halves
    window_img_l = img_blue[w_y1:w_y2, wl_x1:wl_x2]
    window_img_r = img_blue[w_y1:w_y2, wr_x1:wr_x2]
    window_img_l = cv2.GaussianBlur(window_img_l, (5, 5), 20)
    window_img_r = cv2.GaussianBlur(window_img_r, (5, 5), 20)

    filter_img_l = cv2.filter2D(window_img_l, -1, __gabor_kern_diag)
    filter_img_r = cv2.filter2D(window_img_r, -1,
                                cv2.flip(__gabor_kern_diag, 1))
    filter_img = np.concatenate([filter_img_l, filter_img_r], axis=1)

    # In polar image, x <-> theta, y <-> magnitude
    max_vals = np.max(filter_img, axis=0)
    ys = np.argmax(filter_img,
                   axis=0)  # Take highest filter response as limbus points
    xs = (np.arange(filter_img.shape[1]) + wl_x1)[max_vals > __min_thresh]
    ys = (ys + w_y1)[max_vals > __min_thresh]

    l_lid_pts = np.squeeze(np.dstack([xs, ys]), axis=0)

    # Only RANSAC fit eyelid if there are enough points
    if l_lid_pts.size < __min_num_pts_u * 2:
        eyelid_lower_line = None
    else:
        eyelid_lower_line = ransac_line(l_lid_pts)

    if eyelid_lower_line is not None:
        a, b = eyelid_lower_line
        b = b + line_y_offset
        eyelid_lower_line = a, b

    if debug_index:
        debug_img = eye_img.copy()

        filter_img = cv2.cvtColor(filter_img, cv2.COLOR_GRAY2BGR)

        if l_lid_pts.size > 2:
            draw_points(debug_img, l_lid_pts, (0, 0, 255), 1, 2)

        if eyelid_lower_line is not None:
            cv2.line(debug_img, (0, int(b)), (img_w, int(a * img_w + b)),
                     (0, 255, 0))

        window_img = np.concatenate([window_img_l, window_img_r], axis=1)
        stacked_windows = stack_imgs_vertical([window_img, filter_img])
        stacked_imgs = stack_imgs_horizontal([stacked_windows, debug_img])
        __debug_imgs_lower[debug_index] = stacked_imgs

        if debug_index > 2:
            cv2.imshow(__winname + repr(debug_index) + "l", stacked_imgs)

    return eyelid_lower_line
コード例 #5
0
ファイル: eyelid_locator.py プロジェクト: zhoushiwei/EyeTab
def find_upper_eyelid(eye_img, debug_index):

    u_2_win_rats_w = [0.0, 1.0, 0.0]  # Margins around ROI windows
    u_2_win_rats_h = [0.0, 0.5, 0.5]

    # FIXME - using r channel?
    img_blue = cv2.split(eye_img)[2]
    img_w, img_h = eye_img.shape[:2]

    # Indexes to extract window sub-images
    w_y1, w_y2 = int(img_h * u_2_win_rats_h[0]), int(img_h *
                                                     sum(u_2_win_rats_h[:2]))
    w_x1, w_x2 = int(img_w * u_2_win_rats_w[0]), int(img_w *
                                                     sum(u_2_win_rats_w[:2]))

    # Split image into two halves
    window_img = img_blue[w_y1:w_y2, w_x1:w_x2]

    # Supress eyelashes
    morph_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
    window_img = cv2.morphologyEx(window_img, cv2.MORPH_CLOSE, morph_kernel)

    # Filter right half with inverse kernel of left half to ignore iris/sclera boundary
    filter_img_win = cv2.filter2D(window_img, -1, __gabor_kern_horiz)

    # Copy windows back into correct places in full filter image
    filter_img = np.zeros(eye_img.shape[:2], dtype=np.uint8)
    filter_img[w_y1:w_y2, w_x1:w_x2] = filter_img_win

    # Mask with circles
    cv2.circle(filter_img,
               (3 * filter_img.shape[1] / 7, filter_img.shape[0] / 2),
               filter_img.shape[1] / 4, 0, -1)
    cv2.circle(filter_img,
               (4 * filter_img.shape[1] / 7, filter_img.shape[0] / 2),
               filter_img.shape[1] / 4, 0, -1)

    ys = np.argmax(filter_img, axis=0)
    xs = np.arange(filter_img.shape[1])[ys > 0]
    ys = (ys)[ys > 0]

    u_lid_pts = []

    for i, x in enumerate(xs):
        col = filter_img.T[x]
        start_ind, end_ind = ys[i] + 5, min(ys[i] + 100, len(col) - 2)
        col_window = col[start_ind:end_ind]
        max_col = np.max(col)
        max_win = np.max(col_window)
        if max_col - max_win < 50:
            new_y = np.argmax(col_window) + ys[i] + 5
            u_lid_pts.append((x, new_y))
        else:
            u_lid_pts.append((x, ys[i]))

    # Only RANSAC fit eyelid if there are enough points
    if len(u_lid_pts) < __min_num_pts_u * 2:
        eyelid_upper_parabola = None
        u_lid_pts = []
    else:
        u_lid_pts_l = [(x, y) for (x, y) in u_lid_pts
                       if x < filter_img.shape[1] / 2]
        u_lid_pts_r = [(x, y) for (x, y) in u_lid_pts
                       if x > filter_img.shape[1] / 2]

        # Fit eye_img coord points of sclera-segs to degree 2 polynomial
        # a(x^2) + b(x) + c
        eyelid_upper_parabola = ransac_parabola(u_lid_pts_l,
                                                u_lid_pts_r,
                                                ransac_iters_max=5,
                                                refine_iters_max=2,
                                                max_err=4)
    if eyelid_upper_parabola is not None:
        a, b, c = eyelid_upper_parabola
        c = c - __parabola_y_offset
        eyelid_upper_parabola = a, b, c

    # --------------------- Debug Drawing ---------------------
    if debug_index:
        debug_img = eye_img.copy()

        if eyelid_upper_parabola is not None:
            lid_xs = np.arange(21) * img_w / 20
            lid_ys = a * lid_xs**2 + b * lid_xs + c
            lid_pts = np.dstack([lid_xs, lid_ys]).astype(int)
            cv2.polylines(debug_img, lid_pts, False, (0, 255, 0), 1)

        draw_points(debug_img, u_lid_pts, (0, 0, 255), 1, 2)
        filter_img = cv2.cvtColor(filter_img, cv2.COLOR_GRAY2BGR)
        draw_points(filter_img, u_lid_pts, (0, 0, 255), 1, 2)

        stacked_windows = stack_imgs_vertical([window_img, filter_img])
        stacked_imgs = stack_imgs_horizontal([stacked_windows, debug_img])
        __debug_imgs_upper[debug_index] = stacked_imgs

        if debug_index > 2:
            cv2.imshow(__winname + repr(debug_index) + "u", stacked_imgs)
    # --------------------- Debug Drawing ---------------------

    return eyelid_upper_parabola
コード例 #6
0
ファイル: gaze_system.py プロジェクト: zhoushiwei/EyeTab
    def get_gaze_from_frame(self, frame):

        frame = cv2.undistort(frame, cam_mat_n7, dist_coefs_n7)

        frame_pyr = image_utils.make_gauss_pyr(frame, 4)
        full_frame = frame_pyr[1].copy()
        half_frame = frame_pyr[2].copy()

        limbuses = [None, None]
        gaze_pts_mm = [None, None]
        gaze_pts_px = [None, None]

        try:
            sub_img_cx0, sub_img_cy0 = None, None
            eye_r_roi, eye_l_roi = eye_extractor.get_eye_rois(
                frame_pyr, 4, debug=self.debug, device=self.device)

            for i, eye_roi in enumerate([eye_r_roi, eye_l_roi]):

                try:
                    if eye_roi.img is None: break

                    # Gives unique winnames for each ROI
                    debug_index = ((i + 1) if self.debug else False)

                    eye_roi.img = self.pre_proc.erase_specular(
                        eye_roi.img, debug=debug_index)

                    pupil_x0, pupil_y0 = eye_center_locator_combined.find_pupil(
                        eye_roi.img,
                        fast_width_grads=25.0,
                        fast_width_iso=80.0,
                        weight_grads=0.8,
                        weight_iso=0.2,
                        debug_index=debug_index)
                    eye_roi.refine_pupil((pupil_x0, pupil_y0), full_frame)
                    roi_x0, roi_y0, roi_w, roi_h = eye_roi.roi_x0, eye_roi.roi_y0, eye_roi.roi_w, eye_roi.roi_h

                    u_eyelid, l_eyelid = find_eyelids(eye_roi.img, debug_index)

                    pts_found = get_limb_pts(eye_img=eye_roi.img,
                                             phi=20,
                                             angle_step=1,
                                             debug_index=debug_index)
                    pts_found = eyelid_locator.filter_limbus_pts(
                        u_eyelid, l_eyelid, pts_found)

                    ellipse = ransac_ellipse.ransac_ellipse_fit(
                        points=pts_found,
                        bgr_img=eye_roi.img,
                        roi_pos=(roi_x0, roi_y0),
                        ransac_iters_max=5,
                        refine_iters_max=3,
                        max_err=1,
                        debug=False)

                    # Shift 2D limbus ellipse and points to account for eye ROI coords
                    (ell_x0, ell_y0), (ell_w,
                                       ell_h), angle = ellipse.rotated_rect
                    new_rotated_rect = (roi_x0 + ell_x0,
                                        roi_y0 + ell_y0), (ell_w, ell_h), angle
                    ellipse = Ellipse(new_rotated_rect)
                    pts_found_to_draw = [(px + roi_x0, py + roi_y0)
                                         for (px, py) in pts_found]

                    # Correct coords when extracting eye for half-frame
                    (sub_img_cx0, sub_img_cy0) = (roi_x0 + ell_x0,
                                                  roi_y0 + ell_y0)

                    # Ignore incorrect limbus
                    limbus = gaze_geometry.ellipse_to_limbuses_persp_geom(
                        ellipse, self.device)
                    limbuses[i] = limbus

                    # Draw eye features onto debug image
                    draw_utils.draw_limbus(full_frame,
                                           limbus,
                                           color=debug_colors[i],
                                           scale=1)
                    draw_utils.draw_points(full_frame,
                                           pts_found_to_draw,
                                           color=debug_colors[i],
                                           width=1,
                                           thickness=2)
                    draw_utils.draw_normal(full_frame,
                                           limbus,
                                           self.device,
                                           color=debug_colors[i],
                                           scale=1)
                    draw_utils.draw_normal(half_frame,
                                           limbus,
                                           self.device,
                                           color=debug_colors[i],
                                           scale=0.5,
                                           arrow_len_mm=20)
                    eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 +
                                                         eye_roi.roi_h),
                                         eye_roi.roi_x0:(eye_roi.roi_x0 +
                                                         eye_roi.roi_w)]
                    draw_utils.draw_eyelids(u_eyelid, l_eyelid, eye_img)

                except ransac_ellipse.NoEllipseFound:
                    if self.debug: print 'No Ellipse Found'
                    cv2.rectangle(full_frame, (roi_x0, roi_y0),
                                  (roi_x0 + roi_w, roi_y0 + roi_h),
                                  (0, 0, 255),
                                  thickness=4)

                except ransac_ellipse.CoverageTooLow as e:
                    if self.debug:
                        print 'Ellipse Coverage Too Low : %s' % e.msg
                    cv2.rectangle(full_frame, (roi_x0, roi_y0),
                                  (roi_x0 + roi_w, roi_y0 + roi_h),
                                  (0, 0, 255),
                                  thickness=4)

                finally:

                    # Extract only eye_roi block after other drawing methods
                    if sub_img_cx0 is not None:
                        eye_img = full_frame[sub_img_cy0 - 60:sub_img_cy0 + 60,
                                             sub_img_cx0 - 60:sub_img_cx0 + 60]
                    else:
                        eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 +
                                                             eye_roi.roi_h),
                                             eye_roi.roi_x0:(eye_roi.roi_x0 +
                                                             eye_roi.roi_w)]

                    # Transfer eye_img block to section of half_frame
                    half_frame[half_frame.shape[0] -
                               eye_img.shape[0]:half_frame.shape[0],
                               (half_frame.shape[1] - eye_img.shape[1]) *
                               i:half_frame.shape[1] if i else eye_img.
                               shape[1]] = eye_img

        except eye_extractor.NoEyesFound as e:
            if self.debug: print 'No Eyes Found: %s' % e.msg

        # Remove any extreme outliers
        limbuses = limbus_outlier_removal.remove_outliers(limbuses)

        # Get gaze points
        for i, limbus in enumerate(limbuses):
            if limbus is None: continue
            gaze_pts_mm[i] = gaze_geometry.get_gaze_point_mm(limbus)
            gaze_pts_px[i] = gaze_geometry.convert_gaze_pt_mm_to_px(
                gaze_pts_mm[i], self.device)

        smoothed_gaze_pt_mm = self.smoother.smooth_gaze(gaze_pts_mm)
        smoothed_gaze_pt_px = gaze_geometry.convert_gaze_pt_mm_to_px(
            smoothed_gaze_pt_mm, self.device)

        # Visualize in 2D and 3D
        cv2.imshow('gaze system', half_frame)
        self.visualizer3d.update_vis(limbuses, smoothed_gaze_pt_mm)

        # If recording, take a screenshot of vpython and add to vid. capture
        if self.recording:
            vis_screen = self.visualizer3d.take_screenshot()
            stacked_imgs = image_utils.stack_imgs_horizontal(
                [vis_screen, half_frame])
            self.vid_writer.write(stacked_imgs)

        return smoothed_gaze_pt_px
コード例 #7
0
ファイル: eyelid_locator.py プロジェクト: tonysy/EyeTab
def find_lower_eyelid(eye_img, debug_index):

    line_y_offset = 0  # Amount to shift eye-lid by after detection

    img_blue = cv2.split(eye_img)[2]
    img_w, img_h = eye_img.shape[:2]

    # Indexes to extract window sub-images
    w_y1, w_y2 = int(img_h * __l_win_rats_h[0]), int(img_h * sum(__l_win_rats_h[:2]))
    wl_x1, wl_x2 = int(img_w * __l_win_rats_w_l[0]), int(img_w * sum(__l_win_rats_w_l[:2]))
    wr_x1, wr_x2 = int(img_w * __l_win_rats_w_r[0]), int(img_w * sum(__l_win_rats_w_r[:2]))

    # Split image into two halves
    window_img_l = img_blue[w_y1:w_y2, wl_x1:wl_x2]
    window_img_r = img_blue[w_y1:w_y2, wr_x1:wr_x2]
    window_img_l = cv2.GaussianBlur(window_img_l, (5, 5), 20)
    window_img_r = cv2.GaussianBlur(window_img_r, (5, 5), 20)

    filter_img_l = cv2.filter2D(window_img_l, -1, __gabor_kern_diag)
    filter_img_r = cv2.filter2D(window_img_r, -1, cv2.flip(__gabor_kern_diag, 1))
    filter_img = np.concatenate([filter_img_l, filter_img_r], axis=1)

    # In polar image, x <-> theta, y <-> magnitude
    max_vals = np.max(filter_img, axis=0)
    ys = np.argmax(filter_img, axis=0)  # Take highest filter response as limbus points
    xs = (np.arange(filter_img.shape[1]) + wl_x1)[max_vals > __min_thresh]
    ys = (ys + w_y1)[max_vals > __min_thresh]

    l_lid_pts = np.squeeze(np.dstack([xs, ys]), axis=0)

    # Only RANSAC fit eyelid if there are enough points
    if l_lid_pts.size < __min_num_pts_u * 2:
        eyelid_lower_line = None
    else:
        eyelid_lower_line = ransac_line(l_lid_pts)

    if eyelid_lower_line is not None:
        a, b = eyelid_lower_line
        b = b + line_y_offset
        eyelid_lower_line = a, b

    if debug_index:
        debug_img = eye_img.copy()

        filter_img = cv2.cvtColor(filter_img, cv2.COLOR_GRAY2BGR)

        if l_lid_pts.size > 2:
            draw_points(debug_img, l_lid_pts, (0, 0, 255), 1, 2)

        if eyelid_lower_line is not None:
            cv2.line(debug_img, (0, int(b)), (img_w, int(a * img_w + b)), (0, 255, 0))

        window_img = np.concatenate([window_img_l, window_img_r], axis=1)
        stacked_windows = stack_imgs_vertical([window_img, filter_img])
        stacked_imgs = stack_imgs_horizontal([stacked_windows, debug_img])
        __debug_imgs_lower[debug_index] = stacked_imgs

        if debug_index > 2:
            cv2.imshow(__winname + repr(debug_index) + "l", stacked_imgs)

    return eyelid_lower_line
コード例 #8
0
ファイル: eyelid_locator.py プロジェクト: tonysy/EyeTab
def find_upper_eyelid(eye_img, debug_index):

    u_2_win_rats_w = [0.0, 1.0, 0.0]  # Margins around ROI windows
    u_2_win_rats_h = [0.0, 0.5, 0.5]

    # FIXME - using r channel?
    img_blue = cv2.split(eye_img)[2]
    img_w, img_h = eye_img.shape[:2]

    # Indexes to extract window sub-images
    w_y1, w_y2 = int(img_h * u_2_win_rats_h[0]), int(img_h * sum(u_2_win_rats_h[:2]))
    w_x1, w_x2 = int(img_w * u_2_win_rats_w[0]), int(img_w * sum(u_2_win_rats_w[:2]))

    # Split image into two halves
    window_img = img_blue[w_y1:w_y2, w_x1:w_x2]

    # Supress eyelashes
    morph_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
    window_img = cv2.morphologyEx(window_img, cv2.MORPH_CLOSE, morph_kernel)

    # Filter right half with inverse kernel of left half to ignore iris/sclera boundary
    filter_img_win = cv2.filter2D(window_img, -1, __gabor_kern_horiz)

    # Copy windows back into correct places in full filter image
    filter_img = np.zeros(eye_img.shape[:2], dtype=np.uint8)
    filter_img[w_y1:w_y2, w_x1:w_x2] = filter_img_win

    # Mask with circles
    cv2.circle(filter_img, (3 * filter_img.shape[1] / 7, filter_img.shape[0] / 2), filter_img.shape[1] / 4, 0, -1)
    cv2.circle(filter_img, (4 * filter_img.shape[1] / 7, filter_img.shape[0] / 2), filter_img.shape[1] / 4, 0, -1)

    ys = np.argmax(filter_img, axis=0)
    xs = np.arange(filter_img.shape[1])[ys > 0]
    ys = (ys)[ys > 0]

    u_lid_pts = []

    for i, x in enumerate(xs):
        col = filter_img.T[x]
        start_ind, end_ind = ys[i] + 5, min(ys[i] + 100, len(col) - 2)
        col_window = col[start_ind:end_ind]
        max_col = np.max(col)
        max_win = np.max(col_window)
        if max_col - max_win < 50:
            new_y = np.argmax(col_window) + ys[i] + 5
            u_lid_pts.append((x, new_y))
        else:
            u_lid_pts.append((x, ys[i]))

    # Only RANSAC fit eyelid if there are enough points
    if len(u_lid_pts) < __min_num_pts_u * 2:
        eyelid_upper_parabola = None
        u_lid_pts = []
    else:
        u_lid_pts_l = [(x, y) for (x, y) in u_lid_pts if x < filter_img.shape[1] / 2]
        u_lid_pts_r = [(x, y) for (x, y) in u_lid_pts if x > filter_img.shape[1] / 2]

        # Fit eye_img coord points of sclera-segs to degree 2 polynomial
        # a(x^2) + b(x) + c
        eyelid_upper_parabola = ransac_parabola(
            u_lid_pts_l, u_lid_pts_r, ransac_iters_max=5, refine_iters_max=2, max_err=4
        )
    if eyelid_upper_parabola is not None:
        a, b, c = eyelid_upper_parabola
        c = c - __parabola_y_offset
        eyelid_upper_parabola = a, b, c

    # --------------------- Debug Drawing ---------------------
    if debug_index:
        debug_img = eye_img.copy()

        if eyelid_upper_parabola is not None:
            lid_xs = np.arange(21) * img_w / 20
            lid_ys = a * lid_xs ** 2 + b * lid_xs + c
            lid_pts = np.dstack([lid_xs, lid_ys]).astype(int)
            cv2.polylines(debug_img, lid_pts, False, (0, 255, 0), 1)

        draw_points(debug_img, u_lid_pts, (0, 0, 255), 1, 2)
        filter_img = cv2.cvtColor(filter_img, cv2.COLOR_GRAY2BGR)
        draw_points(filter_img, u_lid_pts, (0, 0, 255), 1, 2)

        stacked_windows = stack_imgs_vertical([window_img, filter_img])
        stacked_imgs = stack_imgs_horizontal([stacked_windows, debug_img])
        __debug_imgs_upper[debug_index] = stacked_imgs

        if debug_index > 2:
            cv2.imshow(__winname + repr(debug_index) + "u", stacked_imgs)
    # --------------------- Debug Drawing ---------------------

    return eyelid_upper_parabola
コード例 #9
0
def get_limb_pts(eye_img, phi=20, angle_step=1, debug_index=False):
    
    polar_img_w = 360 / angle_step                                      # Polar image has one column per angle of interest
    phi_range_1 = ((90 - phi) / angle_step, (90 + phi) / angle_step)    # Ranges of angles to be ignored (too close to lids)
    phi_range_2 = ((270 - phi) / angle_step, (270 + phi) / angle_step)
    
    eye_img_grey = cv2.cvtColor(eye_img, cv2.COLOR_BGR2GRAY)      # Do BGR-grey
    eye_img_grey = cv2.medianBlur(eye_img_grey, 5)
    
    # Scale to fixed size image for re-using transform matrix
    scale = eye_img.shape[0] / float(__fixed_width)
    img_fixed_size = cv2.resize(eye_img_grey, (__fixed_width, __fixed_width))
    
    # Transform image into polar coords and blur
    img_polar = linpolar(img_fixed_size, trans_w=polar_img_w, trans_h=__fixed_width / 2)
    img_polar = cv2.GaussianBlur(img_polar, (5, 5), 0)
    
    # Take the segment between min & max radii and filter with Gabor kernel
    img_polar_seg = img_polar[__min_limb_r:__max_limb_r, :]
    filter_img = cv2.filter2D(img_polar_seg, -1, __gabor_kern)
    
    # Black out ignored angles
    filter_img.T[ phi_range_1[0] : phi_range_1[1] ] = 0
    filter_img.T[ phi_range_2[0] : phi_range_2[1] ] = 0

    # In polar image, x <-> theta, y <-> magnitude         
    pol_ys = np.argmax(filter_img, axis=0)                      # Take highest filter response as limbus points
    pol_xs = np.arange(filter_img.shape[1])[pol_ys > 0]
    mags = (pol_ys + __min_limb_r)[pol_ys > 0]
    thts = np.radians(pol_xs * angle_step)

    # Translate each point back into fixed img coords
    xs, ys = cv2.polarToCart(mags.astype(float), thts)
    xs = (xs + __fixed_width / 2) * scale                       # Shift and scale cart. coords back to original eye-ROI coords
    ys = (ys + __fixed_width / 2) * scale
    
    # Points returned in form
    #    [[ x1   y1]
    #     [ x2   y2]
    #         ...
    #     [ xn   yn]]
    pts_cart = np.concatenate([xs, ys], axis=1)
    
    # --------------------- Debug Drawing ---------------------
    if debug_index != False:
        debug_img = eye_img.copy()
        debug_polar = cv2.cvtColor(img_polar, cv2.COLOR_GRAY2BGR)
        
        cv2.imwrite("polar.jpg",debug_polar)
        
        cv2.line(debug_polar, (0, __min_limb_r), (img_polar.shape[1], __min_limb_r), (255, 255, 0))
        cv2.line(debug_polar, (0, __max_limb_r), (img_polar.shape[1], __max_limb_r), (255, 255, 0))
        cv2.circle(debug_img, (debug_img.shape[1] / 2, debug_img.shape[0] / 2), int(debug_img.shape[0] * __limb_r_ratios[0]), (255, 255, 0))
        cv2.circle(debug_img, (debug_img.shape[1] / 2, debug_img.shape[0] / 2), int(debug_img.shape[0] * __limb_r_ratios[1]), (255, 255, 0))
        
        pts_polar = np.squeeze(np.dstack([pol_xs, mags]))
        draw_points(debug_polar, pts_polar, (0, 0, 255), width=1)
        draw_points(debug_img, pts_cart, (0, 0, 255), width=1)
    
        stacked_imgs_polar = stack_imgs_vertical([debug_polar, filter_img])
        stacked_imgs = stack_imgs_horizontal([debug_img, eye_img_grey, stacked_imgs_polar])
        
        __debug_imgs[debug_index] = stacked_imgs
        
        if debug_index == 2:
            full_debug_img = stack_imgs_vertical([__debug_imgs[1], __debug_imgs[2]]);
            cv2.imshow(__winname, full_debug_img)
        elif debug_index > 2:
            cv2.imshow(__winname, stacked_imgs);
    # --------------------- Debug Drawing ---------------------

    return pts_cart
コード例 #10
0
ファイル: gaze_system.py プロジェクト: Cosijopiii/EyeTab
 def get_gaze_from_frame(self, frame):
     
     frame = cv2.undistort(frame, cam_mat_n7, dist_coefs_n7)
     
     frame_pyr = image_utils.make_gauss_pyr(frame, 4)
     full_frame = frame_pyr[1].copy()
     half_frame = frame_pyr[2].copy()
     
     limbuses = [None, None]
     gaze_pts_mm = [None, None]
     gaze_pts_px = [None, None]
     
     try:
         sub_img_cx0, sub_img_cy0 = None, None
         eye_r_roi, eye_l_roi = eye_extractor.get_eye_rois(frame_pyr, 4, debug=self.debug, device=self.device)
         
         for i, eye_roi in enumerate([eye_r_roi, eye_l_roi]):
             
             try:
                 if eye_roi.img is None: break
                 
                 # Gives unique winnames for each ROI
                 debug_index = ((i + 1) if self.debug else False)  
         
                 eye_roi.img = self.pre_proc.erase_specular(eye_roi.img, debug=debug_index)
         
                 pupil_x0, pupil_y0 = eye_center_locator_combined.find_pupil(eye_roi.img,
                                                                             fast_width_grads=25.0,
                                                                             fast_width_iso=80.0,
                                                                             weight_grads=0.8,
                                                                             weight_iso=0.2,
                                                                             debug_index=debug_index)
                 eye_roi.refine_pupil((pupil_x0, pupil_y0), full_frame)
                 roi_x0, roi_y0, roi_w, roi_h = eye_roi.roi_x0, eye_roi.roi_y0, eye_roi.roi_w, eye_roi.roi_h
                 
                 u_eyelid, l_eyelid = find_eyelids(eye_roi.img, debug_index)
                 
                 pts_found = get_limb_pts(eye_img=eye_roi.img,
                                          phi=20,
                                          angle_step=1,
                                          debug_index=debug_index)
                 pts_found = eyelid_locator.filter_limbus_pts(u_eyelid, l_eyelid, pts_found)
                 
                 ellipse = ransac_ellipse.ransac_ellipse_fit(points=pts_found,
                                                             bgr_img=eye_roi.img,
                                                             roi_pos=(roi_x0, roi_y0),
                                                             ransac_iters_max=5,
                                                             refine_iters_max=3,
                                                             max_err=1,
                                                             debug=False)
                 
                 # Shift 2D limbus ellipse and points to account for eye ROI coords
                 (ell_x0, ell_y0), (ell_w, ell_h), angle = ellipse.rotated_rect               
                 new_rotated_rect = (roi_x0 + ell_x0, roi_y0 + ell_y0), (ell_w, ell_h), angle
                 ellipse = Ellipse(new_rotated_rect)                                                                                
                 pts_found_to_draw = [(px + roi_x0, py + roi_y0) for (px, py) in pts_found]
                 
                 # Correct coords when extracting eye for half-frame
                 (sub_img_cx0, sub_img_cy0) = (roi_x0 + ell_x0, roi_y0 + ell_y0)
                 
                 # Ignore incorrect limbus
                 limbus = gaze_geometry.ellipse_to_limbuses_persp_geom(ellipse, self.device)
                 limbuses[i] = limbus
                 
                 # Draw eye features onto debug image
                 draw_utils.draw_limbus(full_frame, limbus, color=debug_colors[i], scale=1)
                 draw_utils.draw_points(full_frame, pts_found_to_draw, color=debug_colors[i], width=1, thickness=2)
                 draw_utils.draw_normal(full_frame, limbus, self.device, color=debug_colors[i], scale=1)
                 draw_utils.draw_normal(half_frame, limbus, self.device, color=debug_colors[i], scale=0.5, arrow_len_mm=20)
                 eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h),
                                      eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)]
                 draw_utils.draw_eyelids(u_eyelid, l_eyelid, eye_img)
                 
             except ransac_ellipse.NoEllipseFound:
                 if self.debug: print 'No Ellipse Found'
                 cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4)
                 
             except ransac_ellipse.CoverageTooLow as e:
                 if self.debug: print 'Ellipse Coverage Too Low : %s' % e.msg
                 cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4)
                 
             finally:
                 
                 # Extract only eye_roi block after other drawing methods
                 if sub_img_cx0 is not None: 
                     eye_img = full_frame[sub_img_cy0 - 60:sub_img_cy0 + 60,
                                          sub_img_cx0 - 60:sub_img_cx0 + 60]
                 else:
                     eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h),
                                          eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)]
                 
                 # Transfer eye_img block to section of half_frame
                 half_frame[half_frame.shape[0] - eye_img.shape[0]:half_frame.shape[0],
                            (half_frame.shape[1] - eye_img.shape[1]) * i: half_frame.shape[1] if i else eye_img.shape[1]] = eye_img
                            
     except eye_extractor.NoEyesFound as e:
         if self.debug: print 'No Eyes Found: %s' % e.msg
         
     # Remove any extreme outliers
     limbuses = limbus_outlier_removal.remove_outliers(limbuses)
     
     # Get gaze points
     for i, limbus in enumerate(limbuses):
         if limbus is None: continue
         gaze_pts_mm[i] = gaze_geometry.get_gaze_point_mm(limbus)
         gaze_pts_px[i] = gaze_geometry.convert_gaze_pt_mm_to_px(gaze_pts_mm[i], self.device)
     
     smoothed_gaze_pt_mm = self.smoother.smooth_gaze(gaze_pts_mm)
     smoothed_gaze_pt_px = gaze_geometry.convert_gaze_pt_mm_to_px(smoothed_gaze_pt_mm, self.device)
     
     # Visualize in 2D and 3D
     cv2.imshow('gaze system', half_frame)
     self.visualizer3d.update_vis(limbuses, smoothed_gaze_pt_mm)
     
     # If recording, take a screenshot of vpython and add to vid. capture
     if self.recording:
         vis_screen = self.visualizer3d.take_screenshot()
         stacked_imgs = image_utils.stack_imgs_horizontal([vis_screen, half_frame])
         self.vid_writer.write(stacked_imgs)
         
     return smoothed_gaze_pt_px
コード例 #11
0
ファイル: ransac_ellipse.py プロジェクト: Cosijopiii/EyeTab
def ransac_ellipse_fit(points, bgr_img, roi_pos, ransac_iters_max=50, refine_iters_max=3, max_err=2, debug=False):
    
    if points.size == 0: raise NoEllipseFound()
    
    blurred_grey_img = cv2.blur(cv2.cvtColor(bgr_img, cv2.COLOR_BGR2GRAY), (3, 3))
    
    image_dx = cv2.Sobel(blurred_grey_img, ddepth=cv2.CV_32F, dx=1, dy=0, ksize=5)
    image_dy = cv2.Sobel(blurred_grey_img, ddepth=cv2.CV_32F, dx=0, dy=1, ksize=5)
    
    pts_x, pts_y = np.split(points, 2, axis=1)
    pts_x, pts_y = np.squeeze(pts_x), np.squeeze(pts_y)
    
    if debug:
        img_points = np.copy(bgr_img)
        draw_points(img_points, points, (0, 0, 255), 1, 2)
        cv2.imshow(winname, img_points)
        cv2.waitKey()
    
    best_ellipse = None
    best_support = float('-inf')
    best_inliers = None

    # Points on right and left of predicted pupil location (center of ROI-img)
    r_inds, l_inds = np.squeeze([pts_x > (bgr_img.shape[1] / 2)]), np.squeeze([pts_x < (bgr_img.shape[1] / 2)])
    
    # Not enough points to start process 
    if r_inds.size < 3 or l_inds.size < 3:  raise NoEllipseFound()  
        
    points_r = points[r_inds]
    points_l = points[l_inds]
    
    if len(points_r) < 3 or len(points_l) < 3:  raise NoEllipseFound()  
    
    # Perform N RANSAC iterations
    for ransac_iter in range(ransac_iters_max):
        
        try:
        
            sample = random.sample(points_r, 3) + (random.sample(points_l, 3))
            ellipse = fit_ellipse(sample, bgr_img.shape[:2])
            
            if debug:
                img_least_sqs = np.copy(bgr_img)
                draw_points(img_least_sqs, points, (0, 0, 255), 1, 2)
                draw_points(img_least_sqs, sample, (0, 255, 255), 6, 2)
                cv2.ellipse(img_least_sqs, ellipse.rotated_rect, (0, 255, 255), 1)
                print 'initial fit: ' + str(ransac_iter + 1)
                cv2.imshow(winname, img_least_sqs)
                cv2.imwrite("ransac_initial" + str(ransac_iter) + ".png", img_least_sqs)
                cv2.waitKey()
            
            # Image-aware sample rejection
            for (p_x, p_y) in sample:
                grad_x, grad_y = ellipse.algebraic_gradient_dir((p_x, p_y))
                dx, dy = image_dx[p_y][p_x], image_dy[p_y][p_x]
                
                # If sample and ellipse gradients don't agree, move to next set of samples
                if dx * grad_x + dy * grad_y <= 0:
                    if debug: print 'Sample and ellipse gradients do not agree, reject'
                    break
            
            else:   # Only continues for else-block did not break on above line (image-aware sample rejection)
                
                # Iteratively refine inliers further
                for _ in range(refine_iters_max):
                    
                    pts_distances = ellipse.distances(pts_x, pts_y)
                    inlier_inds = np.squeeze([np.abs(get_err_scale(ellipse) * pts_distances) < max_err])
                    inliers = points[inlier_inds]
                    
                    if len(inliers) < 5: raise NotEnoughInliers()
                    
                    ellipse = fit_ellipse(inliers, bgr_img.shape[:2])
                    
                    if debug:
                        img_refined = np.copy(bgr_img)
                        draw_points(img_refined, points, (0, 0, 255), 1, 2)
                        draw_points(img_refined, sample, (0, 255, 255), 6, 2)
                        cv2.ellipse(img_refined, ellipse.rotated_rect, (0, 255, 255), 1)
                        draw_points(img_refined, inliers, (0, 255, 0), 1, 2)
                        cv2.imshow(winname, img_refined)
                        cv2.waitKey()
                
                # Calculate the image aware support of the ellipse
                if image_aware_support:
                    
                    inliers_pts_x, inliers_pts_y = np.split(inliers, 2, axis=1)
                    inliers_pts_x, inliers_pts_y = np.squeeze(inliers_pts_x), np.squeeze(inliers_pts_y)
                    
                    # Ellipse gradients at inlier points
                    grads_x, grads_y = ellipse.algebraic_gradient_dirs(inliers_pts_x, inliers_pts_y)
                    
                    # Image gradients at inlier points
                    dxs = image_dx[inliers_pts_y.astype(int), inliers_pts_x.astype(int)]
                    dys = image_dy[inliers_pts_y.astype(int), inliers_pts_x.astype(int)]
                    
                    support = np.sum(dxs.dot(grads_x) + dys.dot(grads_y))
                        
                else: support = len(inliers)
                
                if support > best_support:
                    best_ellipse = ellipse
                    best_support = support
                    best_inliers = inliers
                
                # Early termination for > 95% inliers
                print len(inliers) / float(len(points))
                if len(inliers) / float(len(points)) > 0.95:
                    print "Early Termination"
                    break
                
        except NotEnoughInliers:
            if debug: print 'Not Enough Inliers'
        
        except BadEllipseShape as e:
            if debug: print 'Bad Ellipse Shape: %s' % e.msg
                
    if best_ellipse == None:
        raise NoEllipseFound()

    coverage = calculate_coverage(best_ellipse, best_inliers)
    if coverage < min_coverage:
        raise CoverageTooLow('Minimum inlier coverage: %d, actual coverage: %d' % (min_coverage, coverage))
    
    if debug:
        img_bgr_img = np.copy(bgr_img)
        cv2.ellipse(img_bgr_img, best_ellipse.rotated_rect, (0, 255, 0), 2)
        cv2.imshow(winname, img_bgr_img)
        cv2.waitKey()
    
    return best_ellipse
コード例 #12
0
ファイル: ransac_ellipse.py プロジェクト: zhoushiwei/EyeTab
def ransac_ellipse_fit(points,
                       bgr_img,
                       roi_pos,
                       ransac_iters_max=50,
                       refine_iters_max=3,
                       max_err=2,
                       debug=False):

    if points.size == 0: raise NoEllipseFound()

    blurred_grey_img = cv2.blur(cv2.cvtColor(bgr_img, cv2.COLOR_BGR2GRAY),
                                (3, 3))

    image_dx = cv2.Sobel(blurred_grey_img,
                         ddepth=cv2.CV_32F,
                         dx=1,
                         dy=0,
                         ksize=5)
    image_dy = cv2.Sobel(blurred_grey_img,
                         ddepth=cv2.CV_32F,
                         dx=0,
                         dy=1,
                         ksize=5)

    pts_x, pts_y = np.split(points, 2, axis=1)
    pts_x, pts_y = np.squeeze(pts_x), np.squeeze(pts_y)

    if debug:
        img_points = np.copy(bgr_img)
        draw_points(img_points, points, (0, 0, 255), 1, 2)
        cv2.imshow(winname, img_points)
        cv2.waitKey()

    best_ellipse = None
    best_support = float('-inf')
    best_inliers = None

    # Points on right and left of predicted pupil location (center of ROI-img)
    r_inds, l_inds = np.squeeze([pts_x > (bgr_img.shape[1] / 2)]), np.squeeze(
        [pts_x < (bgr_img.shape[1] / 2)])

    # Not enough points to start process
    if r_inds.size < 3 or l_inds.size < 3: raise NoEllipseFound()

    points_r = points[r_inds]
    points_l = points[l_inds]

    if len(points_r) < 3 or len(points_l) < 3: raise NoEllipseFound()

    # Perform N RANSAC iterations
    for ransac_iter in range(ransac_iters_max):

        try:

            sample = random.sample(points_r, 3) + (random.sample(points_l, 3))
            ellipse = fit_ellipse(sample, bgr_img.shape[:2])

            if debug:
                img_least_sqs = np.copy(bgr_img)
                draw_points(img_least_sqs, points, (0, 0, 255), 1, 2)
                draw_points(img_least_sqs, sample, (0, 255, 255), 6, 2)
                cv2.ellipse(img_least_sqs, ellipse.rotated_rect, (0, 255, 255),
                            1)
                print 'initial fit: ' + str(ransac_iter + 1)
                cv2.imshow(winname, img_least_sqs)
                cv2.imwrite("ransac_initial" + str(ransac_iter) + ".png",
                            img_least_sqs)
                cv2.waitKey()

            # Image-aware sample rejection
            for (p_x, p_y) in sample:
                grad_x, grad_y = ellipse.algebraic_gradient_dir((p_x, p_y))
                dx, dy = image_dx[p_y][p_x], image_dy[p_y][p_x]

                # If sample and ellipse gradients don't agree, move to next set of samples
                if dx * grad_x + dy * grad_y <= 0:
                    if debug:
                        print 'Sample and ellipse gradients do not agree, reject'
                    break

            else:  # Only continues for else-block did not break on above line (image-aware sample rejection)

                # Iteratively refine inliers further
                for _ in range(refine_iters_max):

                    pts_distances = ellipse.distances(pts_x, pts_y)
                    inlier_inds = np.squeeze([
                        np.abs(get_err_scale(ellipse) * pts_distances) <
                        max_err
                    ])
                    inliers = points[inlier_inds]

                    if len(inliers) < 5: raise NotEnoughInliers()

                    ellipse = fit_ellipse(inliers, bgr_img.shape[:2])

                    if debug:
                        img_refined = np.copy(bgr_img)
                        draw_points(img_refined, points, (0, 0, 255), 1, 2)
                        draw_points(img_refined, sample, (0, 255, 255), 6, 2)
                        cv2.ellipse(img_refined, ellipse.rotated_rect,
                                    (0, 255, 255), 1)
                        draw_points(img_refined, inliers, (0, 255, 0), 1, 2)
                        cv2.imshow(winname, img_refined)
                        cv2.waitKey()

                # Calculate the image aware support of the ellipse
                if image_aware_support:

                    inliers_pts_x, inliers_pts_y = np.split(inliers, 2, axis=1)
                    inliers_pts_x, inliers_pts_y = np.squeeze(
                        inliers_pts_x), np.squeeze(inliers_pts_y)

                    # Ellipse gradients at inlier points
                    grads_x, grads_y = ellipse.algebraic_gradient_dirs(
                        inliers_pts_x, inliers_pts_y)

                    # Image gradients at inlier points
                    dxs = image_dx[inliers_pts_y.astype(int),
                                   inliers_pts_x.astype(int)]
                    dys = image_dy[inliers_pts_y.astype(int),
                                   inliers_pts_x.astype(int)]

                    support = np.sum(dxs.dot(grads_x) + dys.dot(grads_y))

                else:
                    support = len(inliers)

                if support > best_support:
                    best_ellipse = ellipse
                    best_support = support
                    best_inliers = inliers

                # Early termination for > 95% inliers
                print len(inliers) / float(len(points))
                if len(inliers) / float(len(points)) > 0.95:
                    print "Early Termination"
                    break

        except NotEnoughInliers:
            if debug: print 'Not Enough Inliers'

        except BadEllipseShape as e:
            if debug: print 'Bad Ellipse Shape: %s' % e.msg

    if best_ellipse == None:
        raise NoEllipseFound()

    coverage = calculate_coverage(best_ellipse, best_inliers)
    if coverage < min_coverage:
        raise CoverageTooLow(
            'Minimum inlier coverage: %d, actual coverage: %d' %
            (min_coverage, coverage))

    if debug:
        img_bgr_img = np.copy(bgr_img)
        cv2.ellipse(img_bgr_img, best_ellipse.rotated_rect, (0, 255, 0), 2)
        cv2.imshow(winname, img_bgr_img)
        cv2.waitKey()

    return best_ellipse