def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() glOrtho( -self.h_pad, (self.frame_count) + self.h_pad, -self.v_pad, 1 + self.v_pad, -1, 1 ) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() color1 = RGBA(.1, .9, .2, 1.) color2 = RGBA(.1, .9, .2, 1.) if self.in_mark != 0 or self.out_mark != self.frame_count: draw_polyline([(self.in_mark, 0), (self.out_mark, 0)], color=color1, thickness=20.) draw_points([ (self.in_mark, 0), ], color=color2, size=20) draw_points([ (self.out_mark, 0), ], color=color2, size=20) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def draw_circle( self, circle_center, circle_normal, circle_radius, color=RGBA(1.1, 0.2, 0.8), num_segments=20, ): vertices = [] vertices.append((0, 0, 0)) # circle center # create circle vertices in the xy plane for i in np.linspace(0.0, 2.0 * math.pi, num_segments): x = math.sin(i) y = math.cos(i) z = 0 vertices.append((x, y, z)) glPushMatrix() glMatrixMode(GL_MODELVIEW) glLoadMatrixf( self.get_pupil_transformation_matrix( circle_normal, circle_center, circle_radius ) ) glutils.draw_polyline( (vertices), color=color, line_type=GL_TRIANGLE_FAN ) # circle glutils.draw_polyline( [(0, 0, 0), (0, 0, 4)], color=RGBA(0, 0, 0), line_type=GL_LINES ) # normal glPopMatrix()
def draw_pupil_outline(pupil_detection_result_2d): """Requires `"ellipse" in pupil_detection_result_2d`""" if pupil_detection_result_2d["confidence"] <= 0.0: return try: pts = cv2.ellipse2Poly( ( int(pupil_detection_result_2d["ellipse"]["center"][0]), int(pupil_detection_result_2d["ellipse"]["center"][1]), ), ( int(pupil_detection_result_2d["ellipse"]["axes"][0] / 2), int(pupil_detection_result_2d["ellipse"]["axes"][1] / 2), ), int(pupil_detection_result_2d["ellipse"]["angle"]), 0, 360, 15, ) except ValueError: # Happens when converting 'nan' to int # TODO: Investigate why results are sometimes 'nan' return confidence = pupil_detection_result_2d["confidence"] * 0.7 draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence)) draw_points( [pupil_detection_result_2d["ellipse"]["center"]], size=20, color=RGBA(1.0, 0.0, 0.0, confidence), sharpness=1.0, )
def _draw_surface_frames(self, surface): if not surface.detected: return ( corners, top_indicator, title_anchor, surface_edit_anchor, marker_edit_anchor, ) = self._get_surface_anchor_points(surface) alpha = min(1, surface.build_up_status) surface_color = rgb_to_rgba(self.color_primary_rgb, alpha=alpha) pyglui_utils.draw_polyline( corners.reshape((5, 2)), color=pyglui_utils.RGBA(*surface_color) ) pyglui_utils.draw_polyline( top_indicator.reshape((4, 2)), color=pyglui_utils.RGBA(*surface_color) ) self._draw_surf_menu( surface, title_anchor, surface_edit_anchor, marker_edit_anchor )
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active or self.visualize: # Draw hand detection results for (x1, y1, x2, y2), fingertips in zip(self.hand_viz, self.finger_viz): pts = np.array([[x1, y1], [x1, y2], [x2, y2], [x2, y1], [x1, y1]], np.int32) cygl_utils.draw_polyline(pts, thickness=3 * self.g_pool.gui_user_scale, color=cygl_utils.RGBA(0., 1., 0., 1.)) for tip in fingertips: if tip is not None: y, x = tip cygl_utils.draw_progress((x, y), 0., 1., inner_radius=25 * self.g_pool.gui_user_scale, outer_radius=35 * self.g_pool.gui_user_scale, color=cygl_utils.RGBA(1., 1., 1., 1.), sharpness=0.9) cygl_utils.draw_points([(x, y)], size=10 * self.g_pool.gui_user_scale, color=cygl_utils.RGBA(1., 1., 1., 1.), sharpness=0.9)
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active: for marker in self.markers: e = marker["ellipses"][-1] # outermost ellipse pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON)
def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() # if self.drag_mode: # color1 = (0.,.8,.5,.5) # color2 = (0.,.8,.5,1.) # else: color1 = (1,1,1,0.4)#(.25,.8,.8,.5) color2 = (1,1,1,1.)#(.25,.8,.8,1.) thickness = 10. draw_polyline(verts=[(0,0),(self.current_frame_index,0)], thickness=thickness,color=RGBA(*color1)) draw_polyline(verts=[(self.current_frame_index,0),(self.frame_count,0)], thickness=thickness,color=RGBA(*color1)) if not self.drag_mode: draw_points([(self.current_frame_index,0)],color=RGBA(*color1),size=30) draw_points([(self.current_frame_index,0)],color=RGBA(*color2),size=20) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def gl_display(self): """ Display marker and surface info inside world screen """ if self.mode == "Show markers and frames": for m in self.markers: hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) draw_polyline(hat.reshape((6,2)),color=RGBA(0.1,1.,1.,.5)) for s in self.surfaces: s.gl_draw_frame(self.img_shape) for s in self.surfaces: if self.locate_3d: s.gl_display_in_window_3d(self.g_pool.image_tex,self.camera_intrinsics) else: s.gl_display_in_window(self.g_pool.image_tex) if self.mode == "Surface edit mode": for s in self.surfaces: s.gl_draw_frame(self.img_shape) s.gl_draw_corners()
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: draw_points_norm([self.smooth_pos],size=15,color=RGBA(1.,1.,0.,.5)) if self.active and self.detected: for e in self.candidate_ellipses: pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])), (int(e[1][0]/2),int(e[1][1]/2)), int(e[-1]),0,360,15) draw_polyline(pts,color=RGBA(0.,1.,0,1.)) # lets draw an indicator on the autostop count e = self.candidate_ellipses[3] pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])), (int(e[1][0]/2),int(e[1][1]/2)), int(e[-1]),0,360,360/self.auto_stop_max) indicator = [e[0]] + pts[self.auto_stop:].tolist() + [e[0]] draw_polyline(indicator,color=RGBA(8.,0.1,0.1,.8),line_type=GL_POLYGON) else: pass
def gl_display_cache_bars(self): """""" padding = 20.0 frame_max = len(self.g_pool.timestamps ) # last marker is garanteed to be frame max. glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() width, height = self.g_pool.camera_render_size h_pad = padding * (frame_max - 2) / float(width) v_pad = padding * 1.0 / (height - 2) gluOrtho( -h_pad, (frame_max - 1) + h_pad, -v_pad, 1 + v_pad, -1, 1 ) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() glTranslatef(0, -0.02, 0) color = (7.0, 0.1, 0.2, 8.0) draw_polyline( self.gl_display_ranges, color=RGBA(*color), line_type=GL_LINES, thickness=2.0, ) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def draw_legend(self, label, width, height, scale): self.glfont.push_state() self.glfont.set_align_string(v_align="right", h_align="top") self.glfont.set_size(15.0 * scale) self.glfont.draw_text(width, 0, label) legend_height = 13.0 * scale pad = 10 * scale self.glfont.draw_text(width / 2, legend_height, "left") self.glfont.draw_text(width, legend_height, "right") self.glfont.pop_state() cygl_utils.draw_polyline( [(pad, 1.5 * legend_height), (width / 4, 1.5 * legend_height)], color=COLOR_LEGEND_EYE_LEFT, line_type=gl.GL_LINES, thickness=4.0 * scale, ) cygl_utils.draw_polyline( [ (width / 2 + pad, 1.5 * legend_height), (width * 3 / 4, 1.5 * legend_height), ], color=COLOR_LEGEND_EYE_RIGHT, line_type=gl.GL_LINES, thickness=4.0 * scale, )
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: draw_points_norm([self.smooth_pos], size=15, color=RGBA(1., 1., 0., .5)) if self.active and self.detected: for e in self.candidate_ellipses: pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15) draw_polyline(pts, color=RGBA(0., 1., 0, 1.)) # lets draw an indicator on the autostop count e = self.candidate_ellipses[3] pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 / self.auto_stop_max) indicator = [e[0]] + pts[self.auto_stop:].tolist() + [e[0]] draw_polyline(indicator, color=RGBA(8., 0.1, 0.1, .8), line_type=GL_POLYGON) else: pass
def gl_display_cache_bars(self): """ """ padding = 20. frame_max = len(self.g_pool.timestamps) #last marker is garanteed to be frame max. glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() width,height = self.win_size h_pad = padding * (frame_max-2)/float(width) v_pad = padding* 1./(height-2) gluOrtho(-h_pad, (frame_max-1)+h_pad, -v_pad, 1+v_pad,-1, 1) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() glTranslatef(0,-.02,0) color = (7.,.1,.2,8.) draw_polyline(self.gl_display_ranges,color=RGBA(*color),line_type=GL_LINES,thickness=2.) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active: for marker in self.markers: e = marker["ellipses"][-1] # outermost ellipse pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0)) if len(self.markers) > 1: draw_polyline( pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON )
def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() glOrtho(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad,-1,1) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() color1 = RGBA(.1,.9,.2,.5) color2 = RGBA(.1,.9,.9,.5) if self.in_mark != 0 or self.out_mark != self.frame_count: draw_polyline( [(self.in_mark,0),(self.out_mark,0)],color=color1,thickness=2) draw_points([(self.in_mark,0),],color=color1,size=10) draw_points([(self.out_mark,0),],color=color1,size=10) if self.sections: for s in self.sections: if self.sections.index(s) != self.focus: draw_polyline( [(s[0],0),(s[1],0)],color=RGBA(.1,.9,.9,.2),thickness=2) for mark in s: draw_points([(mark,0),],color=color2,size=5) if self.mid_sections: for m in self.mid_sections: draw_points([(m,0),],color=RGBA(.1,.9,.9,.1),size=10) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def draw_ellipse( ellipse: Dict, rgba: Tuple, thickness: float, draw_center: bool = False ): try: pts = cv2.ellipse2Poly( center=(int(ellipse["center"][0]), int(ellipse["center"][1])), axes=(int(ellipse["axes"][0] / 2), int(ellipse["axes"][1] / 2)), angle=int(ellipse["angle"]), arcStart=0, arcEnd=360, delta=8, ) except Exception as e: # Known issues: # - There are reports of negative eye_ball axes when drawing the 3D eyeball # outline, which will raise cv2.error. TODO: Investigate cause in detectors. # - There was a case where all values in the ellipse where 'NaN', which raises # ValueError: cannot convert float NaN to integer. TODO: Investigate how we # even got here, since calls to this function are confidence-gated! logger.debug( "Error drawing ellipse! Skipping...\n" f"Ellipse: {ellipse}\n" f"Color: {rgba}\n" f"Error: {type(e)}: {e}" ) return draw_polyline(pts, thickness, RGBA(*rgba)) if draw_center: _draw_circle_filled( tuple(ellipse["center"]), size=3, color=RGBA(*rgba), )
def draw_ellipse(ellipse: Dict, rgba: Tuple, thickness: float, draw_center: bool = False): try: pts = cv2.ellipse2Poly( center=(int(ellipse["center"][0]), int(ellipse["center"][1])), axes=(int(ellipse["axes"][0] / 2), int(ellipse["axes"][1] / 2)), angle=int(ellipse["angle"]), arcStart=0, arcEnd=360, delta=8, ) except Exception as e: # Known issues: # - There are reports of negative eye_ball axes when drawing the 3D eyeball # outline, which will raise cv2.error. TODO: Investigate cause in detectors. logger.debug("Error drawing ellipse! Skipping...\n" f"ellipse: {ellipse}\n" f"{type(e)}: {e}") draw_polyline(pts, thickness, RGBA(*rgba)) if draw_center: draw_points( [ellipse["center"]], size=20, color=RGBA(*rgba), sharpness=1.0, )
def draw_markers(img_size,markers, roi=None): glfont = fontstash.Context() glfont.add_font('opensans',get_opensans_font_path()) if roi != None: glfont.set_size((22 * roi[2]) / float(img_size[1])) else: glfont.set_size(22) glfont.set_color_float((0.2,0.5,0.9,1.0)) for m in markers: #old method using opencv to draw doesn't show the result on screen """ centroid = [m['verts'].sum(axis=0)/4.] origin = m['verts'][0] hat = np.array([[[0,0],[0,1],[.5,1.25],[1,1],[1,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) cv2.polylines(img,np.int0(hat),color = (0,0,255),isClosed=True) cv2.polylines(img,np.int0(centroid),color = (255,255,0),isClosed=True,thickness=2) cv2.putText(img,'id: '+str(m['id']),tuple(np.int0(origin)[0,:]),fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.5, color=(255,100,50)) """ hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) draw_polyline(hat.reshape((6,2)),1,RGBA(1.0,0,0,1.0)) glfont.draw_text(m['verts'][0][0][0],m['verts'][0][0][1],'id: '+str(m['id']))
def gl_display(self): """ This is where we can draw to any gl surface by default this is the main window, below we change that """ # active our window active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self.window) # start drawing things: gl_utils.clear_gl_screen() # set coordinate system to be between 0 and 1 of the extents of the window gl_utils.make_coord_system_norm_based() # draw the image draw_gl_texture(self.img) # make coordinte system identical to the img pixel coordinate system gl_utils.make_coord_system_pixel_based(self.img.shape) # draw some points on top of the image # notice how these show up in our window but not in the main window draw_points([(200, 400), (600, 400)], color=RGBA(0., 4., .8, .8), size=self.my_var) draw_polyline([(200, 400), (600, 400)], color=RGBA(0., 4., .8, .8), thickness=3) # since this is our own window we need to swap buffers in the plugin glfwSwapBuffers(self.window) # and finally reactive the main window glfwMakeContextCurrent(active_window)
def draw_circle(self, circle_center, circle_normal, circle_radius, color=RGBA(1.1, 0.2, .8), num_segments=20): vertices = [] vertices.append((0, 0, 0)) # circle center #create circle vertices in the xy plane for i in np.linspace(0.0, 2.0 * math.pi, num_segments): x = math.sin(i) y = math.cos(i) z = 0 vertices.append((x, y, z)) glPushMatrix() glMatrixMode(GL_MODELVIEW) glLoadMatrixf( self.get_pupil_transformation_matrix(circle_normal, circle_center, circle_radius)) glutils.draw_polyline((vertices), color=color, line_type=GL_TRIANGLE_FAN) # circle glutils.draw_polyline([(0, 0, 0), (0, 0, 4)], color=RGBA(0, 0, 0), line_type=GL_LINES) #normal glPopMatrix()
def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D( -self.h_pad, (self.frame_count) + self.h_pad, -self.v_pad, 1 + self.v_pad ) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() if self.drag_mode: color1 = (0.0, 0.8, 0.5, 0.5) color2 = (0.0, 0.8, 0.5, 1.0) else: color1 = (0.25, 0.8, 0.8, 0.5) color2 = (0.25, 0.8, 0.8, 1.0) draw_polyline(verts=[(0, 0), (self.current_frame_index, 0)], color=RGBA(*color1)) draw_polyline(verts=[(self.current_frame_index, 0), (self.frame_count, 0)], color=RGBA(0.5, 0.5, 0.5, 0.5)) draw_points([(self.current_frame_index, 0)], color=RGBA(*color1), size=40) draw_points([(self.current_frame_index, 0)], color=RGBA(*color2), size=10) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def gl_display(self): """ Display marker and surface info inside world screen """ if self.mode == "Show markers and frames": for m in self.markers: hat = np.array( [[[0, 0], [0, 1], [.5, 1.3], [1, 1], [1, 0], [0, 0]]], dtype=np.float32) hat = cv2.perspectiveTransform(hat, m_marker_to_screen(m)) draw_polyline(hat.reshape((6, 2)), color=RGBA(0.1, 1., 1., .5)) for s in self.surfaces: s.gl_draw_frame(self.img_shape) for s in self.surfaces: if self.locate_3d: s.gl_display_in_window_3d(self.g_pool.image_tex, self.camera_intrinsics) else: s.gl_display_in_window(self.g_pool.image_tex) if self.mode == "Surface edit mode": for s in self.surfaces: s.gl_draw_frame(self.img_shape) s.gl_draw_corners()
def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D( -self.h_pad, (self.frame_count) + self.h_pad, -self.v_pad, 1 + self.v_pad ) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() if self.drag_mode: color1 = (0., .8, .5, .5) color2 = (0., .8, .5, 1.) else: color1 = (.25, .8, .8, .5) color2 = (.25, .8, .8, 1.) draw_polyline(verts=[(0, 0), (self.current_frame_index, 0)], color=RGBA(*color1)) draw_polyline(verts=[(self.current_frame_index, 0), (self.frame_count, 0)], color=RGBA(.5, .5, .5, .5)) draw_points([(self.current_frame_index, 0)], color=RGBA(*color1), size=40) draw_points([(self.current_frame_index, 0)], color=RGBA(*color2), size=10) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def gl_display_in_window_3d(self, world_tex): """ here we map a selected surface onto a seperate window. """ K, img_size = ( self.g_pool.capture.intrinsics.K, self.g_pool.capture.intrinsics.resolution, ) if self._window and self.camera_pose_3d is not None: active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) glClearColor(0.8, 0.8, 0.8, 1.0) glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) glClearDepth(1.0) glDepthFunc(GL_LESS) glEnable(GL_DEPTH_TEST) self.trackball.push() glMatrixMode(GL_MODELVIEW) draw_coordinate_system(l=self.real_world_size["x"]) glPushMatrix() glScalef(self.real_world_size["x"], self.real_world_size["y"], 1) draw_polyline( [[0, 0], [0, 1], [1, 1], [1, 0]], color=RGBA(0.5, 0.3, 0.1, 0.5), thickness=3, ) glPopMatrix() # Draw the world window as projected onto the plane using the homography mapping glPushMatrix() glScalef(self.real_world_size["x"], self.real_world_size["y"], 1) # cv uses 3x3 gl uses 4x4 tranformation matricies m = cvmat_to_glmat(self.m_from_screen) glMultMatrixf(m) glTranslatef(0, 0, -0.01) world_tex.draw() draw_polyline( [[0, 0], [0, 1], [1, 1], [1, 0]], color=RGBA(0.5, 0.3, 0.6, 0.5), thickness=3, ) glPopMatrix() # Draw the camera frustum and origin using the 3d tranformation obtained from solvepnp glPushMatrix() glMultMatrixf(self.camera_pose_3d.T.flatten()) draw_frustum(img_size, K, 150) glLineWidth(1) draw_frustum(img_size, K, 0.1) draw_coordinate_system(l=5) glPopMatrix() self.trackball.pop() glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window)
def gl_display(self): for grid_points in self.img_points: calib_bounds = cv2.convexHull(grid_points)[:,0] #we dont need that extra encapsulation that opencv likes so much draw_polyline(calib_bounds,1,RGBA(0.,0.,1.,.5),line_type=gl.GL_LINE_LOOP) if self._window: self.gl_display_in_window()
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # Draw rectangle preview with threshold color if self.world_size: offset = self.world_size[0] / 130.0 ratio = self.world_size[1] / self.world_size[0] rect_size = None if ratio == 0.75: rect_size = [(self.world_size[0] / 20.0), self.world_size[0] / 20.0 * 1.3] else: rect_size = [(self.world_size[0] / 20.0), self.world_size[0] / 20.0 * 1.0] rect_color = colorsys.hsv_to_rgb(self.color_h / 360, self.color_s / 100, self.color_v / 100) draw_rounded_rect( [offset, self.world_size[1] - rect_size[1] - offset], size=rect_size, corner_radius=offset / 2, color=RGBA(rect_color[0], rect_color[1], rect_color[2], 1.)) if self.active: # Draw contour of hand if self.show_contour: con = [(c[0][0], c[0][1]) for c in self.contour] if len(con) > 2: con.append(con[0]) draw_polyline(con, color=RGBA(0., 1., 0., .7), thickness=5.0) # Draw all detected fingertips if len(self.markers) == 1: marker_norm = normalize( self.markers[0], (self.world_size[0], self.world_size[1]), flip_y=True) draw_points_norm([marker_norm], size=30, color=RGBA(0., 1., 1., .5)) else: for mark in self.markers: marker_norm = normalize( mark, (self.world_size[0], self.world_size[1]), flip_y=True) draw_points_norm([marker_norm], size=30, color=RGBA(0., 0., 1., .5))
def gl_display_in_window_3d(self,world_tex,camera_intrinsics): """ here we map a selected surface onto a seperate window. """ K,dist_coef,img_size = camera_intrinsics['camera_matrix'],camera_intrinsics['dist_coefs'],camera_intrinsics['resolution'] if self._window and self.camera_pose_3d is not None: active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) glClearColor(.8,.8,.8,1.) glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) glClearDepth(1.0) glDepthFunc(GL_LESS) glEnable(GL_DEPTH_TEST) self.trackball.push() glMatrixMode(GL_MODELVIEW) draw_coordinate_system(l=self.real_world_size['x']) glPushMatrix() glScalef(self.real_world_size['x'],self.real_world_size['y'],1) draw_polyline([[0,0],[0,1],[1,1],[1,0]],color = RGBA(.5,.3,.1,.5),thickness=3) glPopMatrix() # Draw the world window as projected onto the plane using the homography mapping glPushMatrix() glScalef(self.real_world_size['x'], self.real_world_size['y'], 1) # cv uses 3x3 gl uses 4x4 tranformation matricies m = cvmat_to_glmat(self.m_from_screen) glMultMatrixf(m) glTranslatef(0,0,-.01) world_tex.draw() draw_polyline([[0,0],[0,1],[1,1],[1,0]],color = RGBA(.5,.3,.6,.5),thickness=3) glPopMatrix() # Draw the camera frustum and origin using the 3d tranformation obtained from solvepnp glPushMatrix() glMultMatrixf(self.camera_pose_3d.T.flatten()) draw_frustum(img_size, K, 150) glLineWidth(1) draw_frustum(img_size, K, .1) draw_coordinate_system(l=5) glPopMatrix() self.trackball.pop() glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) if self.window_should_close: self.close_window()
def gl_display(self): for grid_points in self.img_points: calib_bounds = cv2.convexHull(grid_points)[:,0] #we dont need that extra encapsulation that opencv likes so much draw_polyline(calib_bounds,1,RGBA(0.,0.,1.,.5),line_type=gl.GL_LINE_LOOP) if self._window: self.gl_display_in_window() if self.show_undistortion: gl.glPushMatrix() make_coord_system_norm_based() draw_gl_texture(self.undist_img) gl.glPopMatrix()
def draw_polyline(self, segment: model.Classified_Segment): segment_points = segment.world_2d_points(self._canvas_size) polyline_color = color_from_segment(segment).to_rgba().channels polyline_thickness = 2 if not segment_points: return gl_utils.draw_polyline( verts=segment_points, thickness=float(polyline_thickness), color=gl_utils.RGBA(*polyline_color), ) self.draw_id(segment=segment, ref_point=segment_points[-1])
def _draw_markers(self): color = pyglui_utils.RGBA(*self.color_secondary, 0.5) for marker in self.tracker.markers_unfiltered: hat = np.array( [[[0, 0], [0, 1], [0.5, 1.3], [1, 1], [1, 0], [0, 0]]], dtype=np.float32) hat = cv2.perspectiveTransform( hat, _get_norm_to_points_trans(marker.verts_px)) pyglui_utils.draw_polyline(hat.reshape((6, 2)), color=color) if (marker.perimeter >= self.tracker.marker_min_perimeter and marker.id_confidence > self.tracker.marker_min_confidence): pyglui_utils.draw_polyline(hat.reshape((6, 2)), color=color, line_type=gl.GL_POLYGON)
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: draw_points_norm([self.smooth_pos], size=15, color=RGBA(1., 1., 0., .5)) if self.active and len(self.markers): # draw the largest ellipse of all detected markers for marker in self.markers: e = marker['ellipses'][-1] pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15) draw_polyline(pts, color=RGBA(0., 1., 0, 1.)) if len(self.markers) > 1: draw_polyline(pts, 1, RGBA(1., 0., 0., .5), line_type=GL_POLYGON) # draw indicator on the first detected marker if self.counter and self.markers[0]['marker_type'] == 'Ref': e = self.markers[0]['ellipses'][-1] pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.counter_max) indicator = [e[0]] + pts[self.counter:].tolist()[::-1] + [e[0]] draw_polyline(indicator, color=RGBA(0.1, .5, .7, .8), line_type=GL_POLYGON) # draw indicator on the stop marker(s) if self.auto_stop: for marker in self.markers: if marker['marker_type'] == 'Stop': e = marker['ellipses'][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.auto_stop_max) indicator = [e[0]] + pts[self.auto_stop:].tolist() + [ e[0] ] draw_polyline(indicator, color=RGBA(8., 0.1, 0.1, .8), line_type=GL_POLYGON) else: pass
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active and self.detected: for marker in self.markers: e = marker[-1] # outermost ellipse pts = cv2.ellipse2Poly((int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15) draw_polyline(pts, 1, RGBA(0., 1., 0., 1.))
def _draw_legend_grouped(self, labels, width, height, scale, glfont): glfont.set_size(self.TIMELINE_LINE_HEIGHT * 0.8 * scale) pad = width * 2 / 3 for label in labels: color = self.CMAP[label] glfont.draw_text(width, 0, label) cygl_utils.draw_polyline( [ (pad, self.TIMELINE_LINE_HEIGHT / 2), (width / 4, self.TIMELINE_LINE_HEIGHT / 2), ], color=color, line_type=gl.GL_LINES, thickness=4.0 * scale, ) gl.glTranslatef(0, self.TIMELINE_LINE_HEIGHT * scale, 0)
def draw_eyeball_outline(pupil_detection_result_3d): eye_ball = pupil_detection_result_3d["projected_sphere"] try: pts = cv2.ellipse2Poly( (int(eye_ball["center"][0]), int(eye_ball["center"][1])), (int(eye_ball["axes"][0] / 2), int(eye_ball["axes"][1] / 2)), int(eye_ball["angle"]), 0, 360, 8, ) except ValueError: # Happens when converting 'nan' to int # TODO: Investigate why results are sometimes 'nan' return draw_polyline( pts, 2, RGBA(0.0, 0.9, 0.1, pupil_detection_result_3d["model_confidence"]))
def _draw_markers(self): color = pyglui_utils.RGBA(*self.color_secondary, 0.5) for marker in self.tracker.markers_unfiltered: hat = np.array( [[[0, 0], [0, 1], [0.5, 1.3], [1, 1], [1, 0], [0, 0]]], dtype=np.float32 ) hat = cv2.perspectiveTransform( hat, _get_norm_to_points_trans(marker.verts_px) ) pyglui_utils.draw_polyline(hat.reshape((6, 2)), color=color) if ( marker.perimeter >= self.tracker.marker_min_perimeter and marker.id_confidence > self.tracker.marker_min_confidence ): pyglui_utils.draw_polyline( hat.reshape((6, 2)), color=color, line_type=gl.GL_POLYGON )
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active and self.detected: draw_points_norm([self.smooth_pos],size=15,color=RGBA(1.,1.,0.,.5)) for e in self.candidate_ellipses: pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])), (int(e[1][0]/2),int(e[1][1]/2)), int(e[-1]),0,360,15) draw_polyline(pts,color=RGBA(0.,1.,0,1.)) else: pass
def _draw_markers(self): for marker in self.tracker.markers_unfiltered: color = rgb_to_rgba( SURFACE_MARKER_COLOR_RGB_BY_TYPE[marker.marker_type], alpha=0.5) hat = np.array( [[[0, 0], [0, 1], [0.5, 1.3], [1, 1], [1, 0], [0, 0]]], dtype=np.float32) hat = cv2.perspectiveTransform( hat, _get_norm_to_points_trans(marker.verts_px)) # TODO: Should the had be drawn for small or low confidence markers? pyglui_utils.draw_polyline(hat.reshape((6, 2)), color=pyglui_utils.RGBA(*color)) if (marker.perimeter >= self.tracker.marker_min_perimeter and marker.id_confidence > self.tracker.marker_min_confidence): pyglui_utils.draw_polyline(hat.reshape((6, 2)), color=pyglui_utils.RGBA(*color), line_type=gl.GL_POLYGON)
def _draw_surface_frames(self, surface): if not surface.detected: return corners, top_indicator, title_anchor, surface_edit_anchor, marker_edit_anchor = self._get_surface_anchor_points( surface ) alpha = min(1, surface.build_up_status) pyglui_utils.draw_polyline( corners.reshape((5, 2)), color=pyglui_utils.RGBA(*self.color_primary, alpha) ) pyglui_utils.draw_polyline( top_indicator.reshape((4, 2)), color=pyglui_utils.RGBA(*self.color_primary, alpha), ) self._draw_surf_menu( surface, title_anchor, surface_edit_anchor, marker_edit_anchor )
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ # debug mode within world will show green ellipses around detected ellipses if self.active and self.detected: for e in self.candidate_ellipses: pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])), (int(e[1][0]/2),int(e[1][1]/2)), int(e[-1]),0,360,15) draw_polyline(pts,1,RGBA(0.,1.,0.,1.)) else: pass if self._window: self.gl_display_in_window()
def draw_legend(self, width, height, scale): self.glfont.push_state() self.glfont.set_align_string(v_align='right', h_align='top') self.glfont.set_size(15. * scale) self.glfont.draw_text(width, 0, self.timeline.label) legend_height = 13. * scale pad = 10 * scale self.glfont.draw_text(width, legend_height, 'Activaty') cygl_utils.draw_polyline([(pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3)], color=activity_color, line_type=gl.GL_LINES, thickness=4. * scale) legend_height += 1.5 * pad self.glfont.draw_text(width, legend_height, 'Thresholds') cygl_utils.draw_polyline([(pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3)], color=threshold_color, line_type=gl.GL_LINES, thickness=4. * scale) legend_height += 1.5 * pad self.glfont.draw_text(width, legend_height, 'Blinks') cygl_utils.draw_polyline([(pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3)], color=blink_color, line_type=gl.GL_LINES, thickness=4. * scale)
def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() glOrtho(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad,-1,1) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() color1 = RGBA(.1,.9,.2,1.) color2 = RGBA(.1,.9,.2,1.) if self.in_mark != 0 or self.out_mark != self.frame_count: draw_polyline( [(self.in_mark,0),(self.out_mark,0)],color=color1,thickness=20.) draw_points([(self.in_mark,0),],color=color2,size=20) draw_points([(self.out_mark,0),],color=color2,size=20) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def draw_legend(self, width, height, scale): self.glfont.push_state() self.glfont.set_align_string(v_align="right", h_align="top") self.glfont.set_size(15.0 * scale) legend_height = 13.0 * scale pad = 10 * scale self.glfont.draw_text(width, legend_height, "channel " + str(self.channel + 1)) cygl_utils.draw_polyline( [ (pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3), ], color=COLOR_LEGEND_WORLD, line_type=gl.GL_LINES, thickness=4.0 * scale, ) legend_height += 1.5 * pad self.glfont.pop_state()
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active or self.visualize: # Draw hand detection results for (x1, y1, x2, y2), fingertips in zip(self.hand_viz, self.finger_viz): pts = np.array( [[x1, y1], [x1, y2], [x2, y2], [x2, y1], [x1, y1]], np.int32 ) cygl_utils.draw_polyline( pts, thickness=3 * self.g_pool.gui_user_scale, color=cygl_utils.RGBA(0.0, 1.0, 0.0, 1.0), ) for tip in fingertips: if tip is not None: y, x = tip cygl_utils.draw_progress( (x, y), 0.0, 1.0, inner_radius=25 * self.g_pool.gui_user_scale, outer_radius=35 * self.g_pool.gui_user_scale, color=cygl_utils.RGBA(1.0, 1.0, 1.0, 1.0), sharpness=0.9, ) cygl_utils.draw_points( [(x, y)], size=10 * self.g_pool.gui_user_scale, color=cygl_utils.RGBA(1.0, 1.0, 1.0, 1.0), sharpness=0.9, )
def draw_fps_legend(self, width, height, scale): self.glfont.push_state() self.glfont.set_align_string(v_align="right", h_align="top") self.glfont.set_size(15.0 * scale) self.glfont.draw_text(width, 0, self.fps_timeline.label) legend_height = 13.0 * scale pad = 10 * scale if self.show_world_fps: self.glfont.draw_text(width, legend_height, "world FPS") cygl_utils.draw_polyline( [ (pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3), ], color=COLOR_LEGEND_WORLD, line_type=gl.GL_LINES, thickness=4.0 * scale, ) legend_height += 1.5 * pad if self.show_eye_fps: self.glfont.draw_text(width, legend_height, "eye1 FPS") cygl_utils.draw_polyline( [ (pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3), ], color=COLOR_LEGEND_EYE_LEFT, line_type=gl.GL_LINES, thickness=4.0 * scale, ) legend_height += 1.5 * pad self.glfont.draw_text(width, legend_height, "eye0 FPS") cygl_utils.draw_polyline( [ (pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3), ], color=COLOR_LEGEND_EYE_RIGHT, line_type=gl.GL_LINES, thickness=4.0 * scale, ) self.glfont.pop_state()
def gl_display(self): """ use gl calls to render at least: the published position of the reference better: show the detected postion even if not published """ if self.active: # draw the largest ellipse of all detected markers for marker in self.markers: e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 15, ) draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0)) if len(self.markers) > 1: draw_polyline( pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON ) # draw indicator on the stop marker(s) if self.auto_stop: for marker in self.markers: if marker["marker_type"] == "Stop": e = marker["ellipses"][-1] pts = cv2.ellipse2Poly( (int(e[0][0]), int(e[0][1])), (int(e[1][0] / 2), int(e[1][1] / 2)), int(e[-1]), 0, 360, 360 // self.auto_stop_max, ) indicator = [e[0]] + pts[self.auto_stop :].tolist() + [e[0]] draw_polyline( indicator, color=RGBA(8.0, 0.1, 0.1, 0.8), line_type=gl.GL_POLYGON, )
def draw_legend(self, width, height, scale): self.glfont.push_state() self.glfont.set_align_string(v_align="right", h_align="top") self.glfont.set_size(15.0 * scale) self.glfont.draw_text(width, 0, self.timeline.label) legend_height = 13.0 * scale pad = 10 * scale self.glfont.draw_text(width, legend_height, "Activaty") cygl_utils.draw_polyline( [ (pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3), ], color=activity_color, line_type=gl.GL_LINES, thickness=4.0 * scale, ) legend_height += 1.5 * pad self.glfont.draw_text(width, legend_height, "Thresholds") cygl_utils.draw_polyline( [ (pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3), ], color=threshold_color, line_type=gl.GL_LINES, thickness=4.0 * scale, ) legend_height += 1.5 * pad self.glfont.draw_text(width, legend_height, "Blinks") cygl_utils.draw_polyline( [ (pad, legend_height + pad * 2 / 3), (width / 2, legend_height + pad * 2 / 3), ], color=blink_color, line_type=gl.GL_LINES, thickness=4.0 * scale, )
def gl_display(self): """ Display marker and surface info inside world screen """ if self.mode == "Show Markers and Surfaces": for m in self.markers: hat = np.array([[[0, 0], [0, 1], [0.5, 1.3], [1, 1], [1, 0], [0, 0]]], dtype=np.float32) hat = cv2.perspectiveTransform(hat, m_marker_to_screen(m)) if m["perimeter"] >= self.min_marker_perimeter: draw_polyline(hat.reshape((6, 2)), color=RGBA(0.1, 1.0, 1.0, 0.5)) draw_polyline(hat.reshape((6, 2)), color=RGBA(0.1, 1.0, 1.0, 0.3), line_type=GL_POLYGON) else: draw_polyline(hat.reshape((6, 2)), color=RGBA(0.1, 1.0, 1.0, 0.5)) for s in self.surfaces: if s not in self.edit_surfaces and s is not self.marker_edit_surface: s.gl_draw_frame(self.img_shape) for s in self.edit_surfaces: s.gl_draw_frame(self.img_shape, highlight=True, surface_mode=True) s.gl_draw_corners() if self.marker_edit_surface: inc = [] exc = [] for m in self.markers: if m["perimeter"] >= self.min_marker_perimeter: if self.marker_edit_surface.markers.has_key(m["id"]): inc.append(m["centroid"]) else: exc.append(m["centroid"]) draw_points(exc, size=20, color=RGBA(1.0, 0.5, 0.5, 0.8)) draw_points(inc, size=20, color=RGBA(0.5, 1.0, 0.5, 0.8)) self.marker_edit_surface.gl_draw_frame( self.img_shape, color=(0.0, 0.9, 0.6, 1.0), highlight=True, marker_mode=True ) for s in self.surfaces: if self.locate_3d: s.gl_display_in_window_3d(self.g_pool.image_tex, self.camera_calibration) else: s.gl_display_in_window(self.g_pool.image_tex)
def draw_activation(self, width, height, scale): t0, t1 = self.g_pool.timestamps[0], self.g_pool.timestamps[-1] with gl_utils.Coord_System(t0, t1, -1, 1): cygl_utils.draw_polyline( self.cache["response_points"], color=activity_color, line_type=gl.GL_LINE_STRIP, thickness=scale, ) cygl_utils.draw_polyline( self.cache["class_points"], color=blink_color, line_type=gl.GL_LINE_STRIP, thickness=scale, ) cygl_utils.draw_polyline( self.cache["thresholds"], color=threshold_color, line_type=gl.GL_LINES, thickness=scale, )
def _gl_display_cache_bars(self, width, height, scale): ts = self.g_pool.timestamps with gl_utils.Coord_System(ts[0], ts[-1], height, 0): # Lines for areas that have been cached cached_ranges = [] for r in self.marker_cache.visited_ranges: cached_ranges += ((ts[r[0]], 0), (ts[r[1]], 0)) gl.glTranslatef(0, scale * self.TIMELINE_LINE_HEIGHT / 2, 0) color = pyglui_utils.RGBA(0.8, 0.2, 0.2, 0.8) pyglui_utils.draw_polyline( cached_ranges, color=color, line_type=gl.GL_LINES, thickness=scale * 4 ) cached_ranges = [] for r in self.marker_cache.positive_ranges: cached_ranges += ((ts[r[0]], 0), (ts[r[1]], 0)) color = pyglui_utils.RGBA(0, 0.7, 0.3, 0.8) pyglui_utils.draw_polyline( cached_ranges, color=color, line_type=gl.GL_LINES, thickness=scale * 4 ) # Lines where surfaces have been found in video cached_surfaces = [] for surface in self.surfaces: found_at = [] if surface.location_cache is not None: for r in surface.location_cache.positive_ranges: # [[0,1],[3,4]] found_at += ((ts[r[0]], 0), (ts[r[1]], 0)) cached_surfaces.append(found_at) color = pyglui_utils.RGBA(0, 0.7, 0.3, 0.8) for surface in cached_surfaces: gl.glTranslatef(0, scale * self.TIMELINE_LINE_HEIGHT, 0) pyglui_utils.draw_polyline( surface, color=color, line_type=gl.GL_LINES, thickness=scale * 2 )
def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() # custom events for e in self.custom_events: draw_polyline([(e,.06),(e,.005)], color = RGBA(.8, .8, .8, .8)) size = len(self.custom_events) if size > 1: for i, e in enumerate(self.custom_events): draw_points([(e, .03)], size = 5, color = RGBA(.1, .5, .5, 1.)) i = 0 while True: if i == 0: draw_polyline([(self.custom_events[i],.03),(self.custom_events[i+1],0.03)], color = RGBA(.8, .8, .8, .8)) elif (i > 0) and (i < (size-1)): draw_polyline([(self.custom_events[i] +1,.03),(self.custom_events[i+1],0.03)], color = RGBA(.8, .8, .8, .8)) if 'chain' in self.mode: i += 1 elif 'in out pairs' in self.mode: i += 2 if i > (size-1): break glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def update_window(self, g_pool , gaze_points0 , sphere0 , gaze_points1 = [] , sphere1 = None, intersection_points = [] ): if not self.window: return self.begin_update_window() #sets context self.clear_gl_screen() self.trackball.push() glMatrixMode( GL_MODELVIEW ) # draw things in world camera coordinate system glPushMatrix() glLoadIdentity() calibration_points_line_color = RGBA(0.5,0.5,0.5,0.1); error_line_color = RGBA(1.0,0.0,0.0,0.5) self.draw_coordinate_system(200) if self.world_camera_width != 0: self.draw_frustum( self.world_camera_width/ 10.0 , self.world_camera_height/ 10.0 , self.world_camera_focal / 10.0) for p in self.cal_observed_points_3d: glutils.draw_polyline( [ (0,0,0), p] , 1 , calibration_points_line_color, line_type = GL_LINES) #draw error lines form eye gaze points to ref points for(cal_point,ref_point) in zip(self.cal_ref_points_3d, self.cal_observed_points_3d): glutils.draw_polyline( [ cal_point, ref_point] , 1 , error_line_color, line_type = GL_LINES) #calibration points glutils.draw_points( self.cal_ref_points_3d , 4 , RGBA( 0, 1, 1, 1 ) ) glPopMatrix() if sphere0: # eye camera glPushMatrix() glLoadMatrixf( self.eye_camera_to_world_matrix0.T ) self.draw_coordinate_system(60) self.draw_frustum( self.image_width / 10.0, self.image_height / 10.0, self.focal_length /10.) glPopMatrix() #everything else is in world coordinates #eye sphere_center0 = list(sphere0['center']) sphere_radius0 = sphere0['radius'] self.draw_sphere(sphere_center0,sphere_radius0, color = RGBA(1,1,0,1)) #gazelines for p in self.cal_gaze_points0_3d: glutils.draw_polyline( [ sphere_center0, p] , 1 , calibration_points_line_color, line_type = GL_LINES) #calibration points # glutils.draw_points( self.cal_gaze_points0_3d , 4 , RGBA( 1, 0, 1, 1 ) ) #current gaze points glutils.draw_points( gaze_points0 , 2 , RGBA( 1, 0, 0, 1 ) ) for p in gaze_points0: glutils.draw_polyline( [sphere_center0, p] , 1 , RGBA(0,0,0,1), line_type = GL_LINES) #draw error lines form eye gaze points to ref points for(cal_gaze_point,ref_point) in zip(self.cal_gaze_points0_3d, self.cal_ref_points_3d): glutils.draw_polyline( [ cal_gaze_point, ref_point] , 1 , error_line_color, line_type = GL_LINES) #second eye if sphere1: # eye camera glPushMatrix() glLoadMatrixf( self.eye_camera_to_world_matrix1.T ) self.draw_coordinate_system(60) self.draw_frustum( self.image_width / 10.0, self.image_height / 10.0, self.focal_length /10.) glPopMatrix() #everything else is in world coordinates #eye sphere_center1 = list(sphere1['center']) sphere_radius1 = sphere1['radius'] self.draw_sphere(sphere_center1,sphere_radius1, color = RGBA(1,1,0,1)) #gazelines for p in self.cal_gaze_points1_3d: glutils.draw_polyline( [ sphere_center1, p] , 4 , calibration_points_line_color, line_type = GL_LINES) #calibration points glutils.draw_points( self.cal_gaze_points1_3d , 4 , RGBA( 1, 0, 1, 1 ) ) #current gaze points glutils.draw_points( gaze_points1 , 2 , RGBA( 1, 0, 0, 1 ) ) for p in gaze_points1: glutils.draw_polyline( [sphere_center1, p] , 1 , RGBA(0,0,0,1), line_type = GL_LINES) #draw error lines form eye gaze points to ref points for(cal_gaze_point,ref_point) in zip(self.cal_gaze_points1_3d, self.cal_ref_points_3d): glutils.draw_polyline( [ cal_gaze_point, ref_point] , 1 , error_line_color, line_type = GL_LINES) self.trackball.pop() self.end_update_window() #swap buffers, handle context
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ is_alive = Is_Alive_Manager(is_alive_flag) with is_alive: import logging # Set up root logger for this process before doing imports of logged modules. logger = logging.getLogger() logger.setLevel(logging.INFO) # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w') # fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logger.level+10) # create formatter and add it to the handlers formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) #silence noisy modules logging.getLogger("OpenGL").setLevel(logging.ERROR) # create logger for the context of this function logger = logging.getLogger(__name__) # We deferr the imports becasue of multiprocessing. # Otherwise the world process each process also loads the other imports. #general imports import numpy as np import cv2 #display import glfw from pyglui import ui,graph,cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere import OpenGL.GL as gl from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based from ui_roi import UIRoi #monitoring import psutil import math # helpers/utils from file_methods import Persistent_Dict from version_utils import VersionFormat from methods import normalize, denormalize, Roi, timer from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError from av_writer import JPEG_Writer,AV_Writer # Pupil detectors from pupil_detectors import Detector_2D, Detector_3D pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D} #UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (600,300*eye_id) elif platform.system() == 'Windows': scroll_factor = 1.0 window_position_default = (600,31+300*eye_id) else: scroll_factor = 1.0 window_position_default = (600,300*eye_id) #g_pool holds variables for this process g_pool = Global_Container() # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = version g_pool.app = 'capture' g_pool.pupil_queue = pupil_queue g_pool.timebase = timebase # Callback functions def on_resize(window,w, h): if not g_pool.iconified: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) g_pool.gui.update_window(w,h) graph.adjust_size(w,h) adjust_gl_view(w,h) glfw.glfwMakeContextCurrent(active_window) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key,scancode,action,mods) def on_char(window,char): g_pool.gui.update_char(char) def on_iconify(window,iconified): g_pool.iconified = iconified def on_button(window,button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button,action,mods) def on_pos(window,x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor) if g_pool.u_r.active_edit_pt: pos = normalize((x,y),glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height) ) g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos) def on_scroll(window,x,y): g_pool.gui.update_scroll(x,y*scroll_factor) # load session persistent settings session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id)) if session_settings.get("version",VersionFormat('0.0')) < g_pool.version: logger.info("Session setting are from older version of this app. I will not use those.") session_settings.clear() # Initialize capture cap = autoCreateCapture(cap_src, timebase=g_pool.timebase) default_settings = {'frame_size':(640,480),'frame_rate':60} previous_settings = session_settings.get('capture_settings',None) if previous_settings and previous_settings['name'] == cap.name: cap.settings = previous_settings else: cap.settings = default_settings # Test capture try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return #signal world that we are ready to go # pipe_to_world.send('eye%s process ready'%eye_id) # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.iconified = False g_pool.capture = cap g_pool.flip = session_settings.get('flip',False) g_pool.display_mode = session_settings.get('display_mode','camera_image') g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power", 'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.", 'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."} g_pool.u_r = UIRoi(frame.img.shape) g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get())) def on_frame_size_change(new_size): g_pool.u_r = UIRoi((new_size[1],new_size[0])) cap.on_frame_size_change = on_frame_size_change writer = None pupil_detector_settings = session_settings.get('pupil_detector_settings',None) last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)] g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings) # UI callback functions def set_scale(new_scale): g_pool.gui.scale = new_scale g_pool.gui.collect_menus() def set_display_mode_info(val): g_pool.display_mode = val g_pool.display_mode_info.text = g_pool.display_mode_info_text[val] def set_detector(new_detector): g_pool.pupil_detector.cleanup() g_pool.pupil_detector = new_detector(g_pool) g_pool.pupil_detector.init_gui(g_pool.sidebar) # Initialize glfw glfw.glfwInit() title = "eye %s"%eye_id width,height = session_settings.get('window_size',(frame.width, frame.height)) main_window = glfw.glfwCreateWindow(width,height, title, None, None) window_pos = session_settings.get('window_position',window_position_default) glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() g_pool.image_tex.update_from_frame(frame) glfw.glfwSwapInterval(0) sphere = Sphere(20) #setup GUI g_pool.gui = ui.UI() g_pool.gui.scale = session_settings.get('gui_scale',1) g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left') general_settings = ui.Growing_Menu('General') general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size')) general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) ) general_settings.append(ui.Switch('flip',g_pool,label='Flip image display')) general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") ) g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode]) general_settings.append(g_pool.display_mode_info) g_pool.sidebar.append(general_settings) g_pool.gui.append(g_pool.sidebar) detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method") general_settings.append(detector_selector) # let detector add its GUI g_pool.pupil_detector.init_gui(g_pool.sidebar) # let the camera add its GUI g_pool.capture.init_gui(g_pool.sidebar) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window,on_resize) glfw.glfwSetWindowIconifyCallback(main_window,on_iconify) glfw.glfwSetKeyCallback(main_window,on_key) glfw.glfwSetCharCallback(main_window,on_char) glfw.glfwSetMouseButtonCallback(main_window,on_button) glfw.glfwSetCursorPosCallback(main_window,on_pos) glfw.glfwSetScrollCallback(main_window,on_scroll) #set the last saved window size on_resize(main_window, *glfw.glfwGetWindowSize(main_window)) # load last gui configuration g_pool.gui.configuration = session_settings.get('ui_config',{}) #set up performance graphs pid = os.getpid() ps = psutil.Process(pid) ts = frame.timestamp cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20,130) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140,130) fps_graph.update_rate = 5 fps_graph.label = "%0.0f FPS" #create a timer to control window update frequency window_update_timer = timer(1/60.) def window_should_update(): return next(window_update_timer) # Event loop while not glfw.glfwWindowShouldClose(main_window): if pipe_to_world.poll(): cmd = pipe_to_world.recv() if cmd == 'Exit': break elif cmd == "Ping": pipe_to_world.send("Pong") command = None else: command,payload = cmd if command == 'Set_Detection_Mapping_Mode': if payload == '3d': if not isinstance(g_pool.pupil_detector,Detector_3D): set_detector(Detector_3D) detector_selector.read_only = True else: set_detector(Detector_2D) detector_selector.read_only = False else: command = None # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") cap.seek_to_frame(0) frame = cap.get_frame() #update performace graphs t = frame.timestamp dt,ts = t-ts,t try: fps_graph.add(1./dt) except ZeroDivisionError: pass cpu_graph.update() ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if 'Rec_Start' == command: record_path,raw_mode = payload logger.info("Will save eye video to: %s"%record_path) timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id) if raw_mode and frame.jpeg_buffer: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = JPEG_Writer(video_path,cap.frame_rate) else: video_path = os.path.join(record_path, "eye%s.mp4"%eye_id) writer = AV_Writer(video_path,cap.frame_rate) timestamps = [] elif 'Rec_Stop' == command: logger.info("Done recording.") writer.release() writer = None np.save(timestamps_path,np.asarray(timestamps)) del timestamps if writer: writer.write_video_frame(frame) timestamps.append(frame.timestamp) # pupil ellipse detection result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm') result['id'] = eye_id # stream the result g_pool.pupil_queue.put(result) # GL drawing if window_should_update(): if not g_pool.iconified: glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() # switch to work in normalized coordinate space if g_pool.display_mode == 'algorithm': g_pool.image_tex.update_from_ndarray(frame.img) elif g_pool.display_mode in ('camera_image','roi'): g_pool.image_tex.update_from_ndarray(frame.gray) else: pass make_coord_system_norm_based(g_pool.flip) g_pool.image_tex.draw() window_size = glfw.glfwGetWindowSize(main_window) make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip) if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) ) if result['confidence'] >0: if result.has_key('ellipse'): pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)), int(result['ellipse']['angle']),0,360,15) confidence = result['confidence'] * 0.7 #scale it a little draw_polyline(pts,1,RGBA(1.,0,0,confidence)) draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.) # render graphs graph.push_view() fps_graph.draw() cpu_graph.draw() graph.pop_view() # render GUI g_pool.gui.update() #render the ROI if g_pool.display_mode == 'roi': g_pool.u_r.draw(g_pool.gui.scale) #update screen glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() g_pool.pupil_detector.visualize() #detector decides if we visualize or not # END while running # in case eye recording was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path,np.asarray(timestamps)) glfw.glfwRestoreWindow(main_window) #need to do this for windows os # save session persistent settings session_settings['gui_scale'] = g_pool.gui.scale session_settings['roi'] = g_pool.u_r.get() session_settings['flip'] = g_pool.flip session_settings['display_mode'] = g_pool.display_mode session_settings['ui_config'] = g_pool.gui.configuration session_settings['capture_settings'] = g_pool.capture.settings session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = g_pool.version session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__ session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings() session_settings.close() g_pool.pupil_detector.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() cap.close() logger.debug("Process done")