def remove_outliers(limbuses): limbuses_to_return = [limbuses[0], limbuses[1]] for i, limbus in enumerate(limbuses): # Remove limbuses that point widly out of frame if limbus is None: limbuses_to_return[i] = None continue gaze_pts_mm_x, gaze_pts_mm_y = gaze_geometry.get_gaze_point_mm(limbus) if gaze_pts_mm_x < -screen_w_mm_n7 or gaze_pts_mm_x > screen_w_mm_n7: limbuses_to_return[i] = None continue elif gaze_pts_mm_y < (-screen_h_mm_n7 * 3 / 2) or (gaze_pts_mm_y > 0): limbuses_to_return[i] = None continue else: limbuses_to_return[i] = limbus # Remove limbuses that are too far from eachother (Pupilary Distance) if limbuses_to_return[0] is not None and limbuses_to_return[1] is not None: l1x, l1y, l1z = limbuses_to_return[0].center_mm l2x, l2y, l2z = limbuses_to_return[1].center_mm limb_dist = math.sqrt((l1x - l2x)**2 + (l1y - l2y)**2 + (l1z - l2z)**2) last_x, last_y, last_z = __last_pos if limb_dist > 80: dist_1 = (last_x - l1x)**2 + (last_y - l1y)**2 + (last_z - l1z)**2 dist_2 = (last_x - l2x)**2 + (last_y - l2y)**2 + (last_z - l2z)**2 if dist_1 < dist_2: limbuses_to_return[1] = None else: limbuses_to_return[0] = None else: __last_pos[0], __last_pos[1], __last_pos[2] = (l1x + l2x) / 2, ( l1y + l2y) / 2, (l1z + l2z) / 2 return limbuses_to_return
def update_vis(self, limbuses, smoothed_gaze_pt_mm=None): self.handle_keyboard() for i, limbus in enumerate(limbuses): # Change colors, continue to next eye if limbus is none (not found) limb_color = cvt_col(debug_colors[3 if limbus is None else i]) self.limb_circles[i].color = limb_color self.limb_normals[i].color = limb_color if limbus is None: continue # Show 3D limbus circle self.limb_circles[i].pos = cvt_pt(*limbus.center_mm) self.limb_circles[i].axis = cvt_pt(*limbus.normal) self.limb_circles[i].length = 0.1 # Move eye-sphere self.eye_spheres[i].pos = self.limb_circles[i].pos - self.limb_circles[i].axis * 100 # Show intersection with normal on screen plane gaze_pt_x_mm, gaze_pt_y_mm = gaze_geometry.get_gaze_point_mm(limbus) # Either show long or short limb normal self.limb_normals[i].pos = [cvt_pt(*limbus.center_mm), cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0) if self.vis_mode == 0 else (self.limb_circles[i].pos + self.limb_circles[i].axis * 200)] # Show raw gaze points and trail self.raw_gaze_pts[i].pos = cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0.01) self.raw_gaze_pts[i].trail_object.color = self.raw_gaze_pts_trail_colors[i] # Label eye position # self.limb_labels[i].pos = self.limb_circles[i].pos # self.limb_labels[i].text = '%0.2d, %0.2d, %0.2d' % limbus.center_mm # Handle smoothed gaze pt and save evaluation data if smoothed_gaze_pt_mm is not None: sm_gaze_pt_x, sm_gaze_pt_y = smoothed_gaze_pt_mm self.smoothed_gaze_pt.pos = cvt_pt(sm_gaze_pt_x, sm_gaze_pt_y, 0) self.smoothed_gaze_pt.trail_object.color = self.smoothed_gaze_pt_trail_colors limb_mid_pt = (self.limb_circles[0].pos + self.limb_circles[1].pos) / 2 end_pt = self.smoothed_gaze_pt.pos if self.vis_mode == 1 else limb_mid_pt smoothed_vector_pts = [(1 - a) * limb_mid_pt + a * end_pt for a in np.arange(0, 1.05, 0.05)] self.smoothed_gaze_vector.pos = smoothed_vector_pts self.smoothed_gaze_vector.color = self.smoothed_gaze_pt_trail_colors # --------------------------------- # SAVE EVALUATION DATA # --------------------------------- if self.eval_file is not None: # Only record gaze-data after short cooldown if self.marker_switch_cooldown <= 0: correct_pos = self.markers[self.active_marker_vpy_id].pos gaze_pos = self.smoothed_gaze_pt.pos dist_offset = mag(correct_pos - gaze_pos) x_offset = abs(correct_pos.x - gaze_pos.x) correct_vec = correct_pos - limb_mid_pt gaze_vec = gaze_pos - limb_mid_pt angle_offset = math.degrees(correct_vec.diff_angle(gaze_vec)) # marker x,y, gaze x,y, dist to eye-pair, dist_err, dist_err_x, angle_err line_to_write = ','.join([str(x) for x in [self.active_marker_ind, ("%.4f" % correct_pos.x), ("%.4f" % correct_pos.y), ("%.4f" % gaze_pos.x), ("%.4f" % gaze_pos.y), ("%.4f" % mag(limb_mid_pt)), ("%.4f" % dist_offset), ("%.4f" % x_offset), ("%.4f" % angle_offset)]]) self.eval_file.write(line_to_write + '\n') else: self.marker_switch_cooldown -= 1
def get_gaze_from_frame(self, frame): frame = cv2.undistort(frame, cam_mat_n7, dist_coefs_n7) frame_pyr = image_utils.make_gauss_pyr(frame, 4) full_frame = frame_pyr[1].copy() half_frame = frame_pyr[2].copy() limbuses = [None, None] gaze_pts_mm = [None, None] gaze_pts_px = [None, None] try: sub_img_cx0, sub_img_cy0 = None, None eye_r_roi, eye_l_roi = eye_extractor.get_eye_rois( frame_pyr, 4, debug=self.debug, device=self.device) for i, eye_roi in enumerate([eye_r_roi, eye_l_roi]): try: if eye_roi.img is None: break # Gives unique winnames for each ROI debug_index = ((i + 1) if self.debug else False) eye_roi.img = self.pre_proc.erase_specular( eye_roi.img, debug=debug_index) pupil_x0, pupil_y0 = eye_center_locator_combined.find_pupil( eye_roi.img, fast_width_grads=25.0, fast_width_iso=80.0, weight_grads=0.8, weight_iso=0.2, debug_index=debug_index) eye_roi.refine_pupil((pupil_x0, pupil_y0), full_frame) roi_x0, roi_y0, roi_w, roi_h = eye_roi.roi_x0, eye_roi.roi_y0, eye_roi.roi_w, eye_roi.roi_h u_eyelid, l_eyelid = find_eyelids(eye_roi.img, debug_index) pts_found = get_limb_pts(eye_img=eye_roi.img, phi=20, angle_step=1, debug_index=debug_index) pts_found = eyelid_locator.filter_limbus_pts( u_eyelid, l_eyelid, pts_found) ellipse = ransac_ellipse.ransac_ellipse_fit( points=pts_found, bgr_img=eye_roi.img, roi_pos=(roi_x0, roi_y0), ransac_iters_max=5, refine_iters_max=3, max_err=1, debug=False) # Shift 2D limbus ellipse and points to account for eye ROI coords (ell_x0, ell_y0), (ell_w, ell_h), angle = ellipse.rotated_rect new_rotated_rect = (roi_x0 + ell_x0, roi_y0 + ell_y0), (ell_w, ell_h), angle ellipse = Ellipse(new_rotated_rect) pts_found_to_draw = [(px + roi_x0, py + roi_y0) for (px, py) in pts_found] # Correct coords when extracting eye for half-frame (sub_img_cx0, sub_img_cy0) = (roi_x0 + ell_x0, roi_y0 + ell_y0) # Ignore incorrect limbus limbus = gaze_geometry.ellipse_to_limbuses_persp_geom( ellipse, self.device) limbuses[i] = limbus # Draw eye features onto debug image draw_utils.draw_limbus(full_frame, limbus, color=debug_colors[i], scale=1) draw_utils.draw_points(full_frame, pts_found_to_draw, color=debug_colors[i], width=1, thickness=2) draw_utils.draw_normal(full_frame, limbus, self.device, color=debug_colors[i], scale=1) draw_utils.draw_normal(half_frame, limbus, self.device, color=debug_colors[i], scale=0.5, arrow_len_mm=20) eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h), eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)] draw_utils.draw_eyelids(u_eyelid, l_eyelid, eye_img) except ransac_ellipse.NoEllipseFound: if self.debug: print 'No Ellipse Found' cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4) except ransac_ellipse.CoverageTooLow as e: if self.debug: print 'Ellipse Coverage Too Low : %s' % e.msg cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4) finally: # Extract only eye_roi block after other drawing methods if sub_img_cx0 is not None: eye_img = full_frame[sub_img_cy0 - 60:sub_img_cy0 + 60, sub_img_cx0 - 60:sub_img_cx0 + 60] else: eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h), eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)] # Transfer eye_img block to section of half_frame half_frame[half_frame.shape[0] - eye_img.shape[0]:half_frame.shape[0], (half_frame.shape[1] - eye_img.shape[1]) * i:half_frame.shape[1] if i else eye_img. shape[1]] = eye_img except eye_extractor.NoEyesFound as e: if self.debug: print 'No Eyes Found: %s' % e.msg # Remove any extreme outliers limbuses = limbus_outlier_removal.remove_outliers(limbuses) # Get gaze points for i, limbus in enumerate(limbuses): if limbus is None: continue gaze_pts_mm[i] = gaze_geometry.get_gaze_point_mm(limbus) gaze_pts_px[i] = gaze_geometry.convert_gaze_pt_mm_to_px( gaze_pts_mm[i], self.device) smoothed_gaze_pt_mm = self.smoother.smooth_gaze(gaze_pts_mm) smoothed_gaze_pt_px = gaze_geometry.convert_gaze_pt_mm_to_px( smoothed_gaze_pt_mm, self.device) # Visualize in 2D and 3D cv2.imshow('gaze system', half_frame) self.visualizer3d.update_vis(limbuses, smoothed_gaze_pt_mm) # If recording, take a screenshot of vpython and add to vid. capture if self.recording: vis_screen = self.visualizer3d.take_screenshot() stacked_imgs = image_utils.stack_imgs_horizontal( [vis_screen, half_frame]) self.vid_writer.write(stacked_imgs) return smoothed_gaze_pt_px
def get_gaze_from_frame(self, frame): frame = cv2.undistort(frame, cam_mat_n7, dist_coefs_n7) frame_pyr = image_utils.make_gauss_pyr(frame, 4) full_frame = frame_pyr[1].copy() half_frame = frame_pyr[2].copy() limbuses = [None, None] gaze_pts_mm = [None, None] gaze_pts_px = [None, None] try: sub_img_cx0, sub_img_cy0 = None, None eye_r_roi, eye_l_roi = eye_extractor.get_eye_rois(frame_pyr, 4, debug=self.debug, device=self.device) for i, eye_roi in enumerate([eye_r_roi, eye_l_roi]): try: if eye_roi.img is None: break # Gives unique winnames for each ROI debug_index = ((i + 1) if self.debug else False) eye_roi.img = self.pre_proc.erase_specular(eye_roi.img, debug=debug_index) pupil_x0, pupil_y0 = eye_center_locator_combined.find_pupil(eye_roi.img, fast_width_grads=25.0, fast_width_iso=80.0, weight_grads=0.8, weight_iso=0.2, debug_index=debug_index) eye_roi.refine_pupil((pupil_x0, pupil_y0), full_frame) roi_x0, roi_y0, roi_w, roi_h = eye_roi.roi_x0, eye_roi.roi_y0, eye_roi.roi_w, eye_roi.roi_h u_eyelid, l_eyelid = find_eyelids(eye_roi.img, debug_index) pts_found = get_limb_pts(eye_img=eye_roi.img, phi=20, angle_step=1, debug_index=debug_index) pts_found = eyelid_locator.filter_limbus_pts(u_eyelid, l_eyelid, pts_found) ellipse = ransac_ellipse.ransac_ellipse_fit(points=pts_found, bgr_img=eye_roi.img, roi_pos=(roi_x0, roi_y0), ransac_iters_max=5, refine_iters_max=3, max_err=1, debug=False) # Shift 2D limbus ellipse and points to account for eye ROI coords (ell_x0, ell_y0), (ell_w, ell_h), angle = ellipse.rotated_rect new_rotated_rect = (roi_x0 + ell_x0, roi_y0 + ell_y0), (ell_w, ell_h), angle ellipse = Ellipse(new_rotated_rect) pts_found_to_draw = [(px + roi_x0, py + roi_y0) for (px, py) in pts_found] # Correct coords when extracting eye for half-frame (sub_img_cx0, sub_img_cy0) = (roi_x0 + ell_x0, roi_y0 + ell_y0) # Ignore incorrect limbus limbus = gaze_geometry.ellipse_to_limbuses_persp_geom(ellipse, self.device) limbuses[i] = limbus # Draw eye features onto debug image draw_utils.draw_limbus(full_frame, limbus, color=debug_colors[i], scale=1) draw_utils.draw_points(full_frame, pts_found_to_draw, color=debug_colors[i], width=1, thickness=2) draw_utils.draw_normal(full_frame, limbus, self.device, color=debug_colors[i], scale=1) draw_utils.draw_normal(half_frame, limbus, self.device, color=debug_colors[i], scale=0.5, arrow_len_mm=20) eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h), eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)] draw_utils.draw_eyelids(u_eyelid, l_eyelid, eye_img) except ransac_ellipse.NoEllipseFound: if self.debug: print 'No Ellipse Found' cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4) except ransac_ellipse.CoverageTooLow as e: if self.debug: print 'Ellipse Coverage Too Low : %s' % e.msg cv2.rectangle(full_frame, (roi_x0, roi_y0), (roi_x0 + roi_w, roi_y0 + roi_h), (0, 0, 255), thickness=4) finally: # Extract only eye_roi block after other drawing methods if sub_img_cx0 is not None: eye_img = full_frame[sub_img_cy0 - 60:sub_img_cy0 + 60, sub_img_cx0 - 60:sub_img_cx0 + 60] else: eye_img = full_frame[eye_roi.roi_y0:(eye_roi.roi_y0 + eye_roi.roi_h), eye_roi.roi_x0:(eye_roi.roi_x0 + eye_roi.roi_w)] # Transfer eye_img block to section of half_frame half_frame[half_frame.shape[0] - eye_img.shape[0]:half_frame.shape[0], (half_frame.shape[1] - eye_img.shape[1]) * i: half_frame.shape[1] if i else eye_img.shape[1]] = eye_img except eye_extractor.NoEyesFound as e: if self.debug: print 'No Eyes Found: %s' % e.msg # Remove any extreme outliers limbuses = limbus_outlier_removal.remove_outliers(limbuses) # Get gaze points for i, limbus in enumerate(limbuses): if limbus is None: continue gaze_pts_mm[i] = gaze_geometry.get_gaze_point_mm(limbus) gaze_pts_px[i] = gaze_geometry.convert_gaze_pt_mm_to_px(gaze_pts_mm[i], self.device) smoothed_gaze_pt_mm = self.smoother.smooth_gaze(gaze_pts_mm) smoothed_gaze_pt_px = gaze_geometry.convert_gaze_pt_mm_to_px(smoothed_gaze_pt_mm, self.device) # Visualize in 2D and 3D cv2.imshow('gaze system', half_frame) self.visualizer3d.update_vis(limbuses, smoothed_gaze_pt_mm) # If recording, take a screenshot of vpython and add to vid. capture if self.recording: vis_screen = self.visualizer3d.take_screenshot() stacked_imgs = image_utils.stack_imgs_horizontal([vis_screen, half_frame]) self.vid_writer.write(stacked_imgs) return smoothed_gaze_pt_px
def update_vis(self, limbuses, smoothed_gaze_pt_mm=None): self.handle_keyboard() for i, limbus in enumerate(limbuses): # Change colors, continue to next eye if limbus is none (not found) limb_color = cvt_col(debug_colors[3 if limbus is None else i]) self.limb_circles[i].color = limb_color self.limb_normals[i].color = limb_color if limbus is None: continue # Show 3D limbus circle self.limb_circles[i].pos = cvt_pt(*limbus.center_mm) self.limb_circles[i].axis = cvt_pt(*limbus.normal) self.limb_circles[i].length = 0.1 # Move eye-sphere self.eye_spheres[i].pos = self.limb_circles[ i].pos - self.limb_circles[i].axis * 100 # Show intersection with normal on screen plane gaze_pt_x_mm, gaze_pt_y_mm = gaze_geometry.get_gaze_point_mm( limbus) # Either show long or short limb normal self.limb_normals[i].pos = [ cvt_pt(*limbus.center_mm), cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0) if self.vis_mode == 0 else (self.limb_circles[i].pos + self.limb_circles[i].axis * 200) ] # Show raw gaze points and trail self.raw_gaze_pts[i].pos = cvt_pt(gaze_pt_x_mm, gaze_pt_y_mm, 0.01) self.raw_gaze_pts[ i].trail_object.color = self.raw_gaze_pts_trail_colors[i] # Label eye position # self.limb_labels[i].pos = self.limb_circles[i].pos # self.limb_labels[i].text = '%0.2d, %0.2d, %0.2d' % limbus.center_mm # Handle smoothed gaze pt and save evaluation data if smoothed_gaze_pt_mm is not None: sm_gaze_pt_x, sm_gaze_pt_y = smoothed_gaze_pt_mm self.smoothed_gaze_pt.pos = cvt_pt(sm_gaze_pt_x, sm_gaze_pt_y, 0) self.smoothed_gaze_pt.trail_object.color = self.smoothed_gaze_pt_trail_colors limb_mid_pt = (self.limb_circles[0].pos + self.limb_circles[1].pos) / 2 end_pt = self.smoothed_gaze_pt.pos if self.vis_mode == 1 else limb_mid_pt smoothed_vector_pts = [(1 - a) * limb_mid_pt + a * end_pt for a in np.arange(0, 1.05, 0.05)] self.smoothed_gaze_vector.pos = smoothed_vector_pts self.smoothed_gaze_vector.color = self.smoothed_gaze_pt_trail_colors # --------------------------------- # SAVE EVALUATION DATA # --------------------------------- if self.eval_file is not None: # Only record gaze-data after short cooldown if self.marker_switch_cooldown <= 0: correct_pos = self.markers[self.active_marker_vpy_id].pos gaze_pos = self.smoothed_gaze_pt.pos dist_offset = mag(correct_pos - gaze_pos) x_offset = abs(correct_pos.x - gaze_pos.x) correct_vec = correct_pos - limb_mid_pt gaze_vec = gaze_pos - limb_mid_pt angle_offset = math.degrees( correct_vec.diff_angle(gaze_vec)) # marker x,y, gaze x,y, dist to eye-pair, dist_err, dist_err_x, angle_err line_to_write = ','.join([ str(x) for x in [ self.active_marker_ind, ("%.4f" % correct_pos.x), ("%.4f" % correct_pos.y), ("%.4f" % gaze_pos.x), ("%.4f" % gaze_pos.y), ("%.4f" % mag(limb_mid_pt)), ("%.4f" % dist_offset), ("%.4f" % x_offset), ("%.4f" % angle_offset) ] ]) self.eval_file.write(line_to_write + '\n') else: self.marker_switch_cooldown -= 1