def update(self,frame,recent_pupil_positions,events): if frame.index == self.out_mark or frame.index == self.in_mark: self.g_pool.play=False if self.drag_in: x,y = glfwGetCursorPos(glfwGetCurrentContext()) x,_ = self.screen_to_bar_space((x,y)) self.in_mark = x elif self.drag_out: x,y = glfwGetCursorPos(glfwGetCurrentContext()) x,_ = self.screen_to_bar_space((x,y)) self.out_mark = x
def update(self, frame, events): if frame.index == self.out_mark or frame.index == self.in_mark: self.g_pool.play = False if self.drag_in: x, y = glfwGetCursorPos(glfwGetCurrentContext()) x, _ = self.screen_to_bar_space((x, y)) self.in_mark = x elif self.drag_out: x, y = glfwGetCursorPos(glfwGetCurrentContext()) x, _ = self.screen_to_bar_space((x, y)) self.out_mark = x
def on_click(self,img_pos,button,action): """ gets called when the user clicks in the window screen """ hdpi_factor = float(glfwGetFramebufferSize(glfwGetCurrentContext())[0]/glfwGetWindowSize(glfwGetCurrentContext())[0]) pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = pos[0]*hdpi_factor,pos[1]*hdpi_factor #drag the seek point if action == GLFW_PRESS: screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0)) screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0)) #in mark dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20: self.drag_in=True return #out mark dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20: self.drag_out=True elif action == GLFW_RELEASE: if self.drag_out or self.drag_in: logger.info("Section: "+self.get_string()) self.drag_out = False self.drag_in = False
def visualize(self, frame, alpha, scale, show_ellipses, pupil_positions): if not self.initialized: return requested_eye_frame_idx = self.eye_world_frame_map[frame.index] # 1. do we need a new frame? if requested_eye_frame_idx != self.current_eye_frame.index: if requested_eye_frame_idx == self.source.get_frame_index() + 2: # if we just need to seek by one frame, its faster to just read one and and throw it away. self.source.get_frame() if requested_eye_frame_idx != self.source.get_frame_index() + 1: self.source.seek_to_frame(requested_eye_frame_idx) try: self.current_eye_frame = self.source.get_frame() except EndofVideoError: logger.info("Reached the end of the eye video for eye video {}.".format(self.eyeid)) # 2. dragging image if self.drag_offset is not None: x, y = glfwGetCursorPos(glfwGetCurrentContext()) pos = x * self.hdpi_fac, y * self.hdpi_fac pos = normalize(pos, self.g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) self.pos = int(pos[0] + self.drag_offset[0]), int(pos[1] + self.drag_offset[1]) # 3. keep in image bounds, do this even when not dragging because the image video_sizes could change. video_size = round(self.current_eye_frame.width * scale), round(self.current_eye_frame.height * scale) # frame.img.shape[0] is height, frame.img.shape[1] is width of screen self.pos = (min(frame.img.shape[1] - video_size[0], max(self.pos[0], 0)), min(frame.img.shape[0] - video_size[1], max(self.pos[1], 0))) # 4. vflipping images, converting to greyscale eyeimage = self.current_eye_frame.gray eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR) if show_ellipses: try: pp = next((pp for pp in pupil_positions if pp['id'] == self.eyeid and pp['timestamp'] == self.current_eye_frame.timestamp)) except StopIteration: pass else: el = pp['ellipse'] conf = int(pp.get('model_confidence', pp.get('confidence', 0.1)) * 255) el_points = getEllipsePts((el['center'], el["axes"], el['angle'])) cv2.polylines(eyeimage, [np.asarray(el_points,dtype='i')], True, (0, 0, 255, conf), thickness=1) cv2.circle(eyeimage,(int(el['center'][0]),int(el['center'][1])), 5, (0, 0, 255, conf), thickness=-1) #flip and scale eyeimage = cv2.resize(eyeimage, (0, 0), fx=scale, fy=scale) if self.hflip: eyeimage = np.fliplr(eyeimage) if self.vflip: eyeimage = np.flipud(eyeimage) transparent_image_overlay(self.pos, eyeimage, frame.img, alpha)
def on_click(self,img_pos,button,action): """ gets called when the user clicks in the window screen """ pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: screen_seek_pos = self.seek_bar_to_screen((self.current_frame_index,0)) dist = abs(pos[0]-screen_seek_pos[0])+abs(pos[1]-screen_seek_pos[1]) if dist < 20: self.drag_mode=True self.was_playing = self.g_pool.play self.g_pool.play = False elif action == GLFW_RELEASE: if self.drag_mode: x, _ = self.screen_to_seek_bar(pos) x = int(min(self.frame_count-5,max(0,x))) try: self.cap.seek_to_frame(x) except: pass self.g_pool.new_seek = True self.drag_mode=False self.g_pool.play = self.was_playing
def recent_events(self, events): frame = events.get("frame") if not frame: return if self.drag_offset is not None: pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos, glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0]) ) # Position in img pixels self.pos[0] = pos[0] + self.drag_offset[0] self.pos[1] = pos[1] + self.drag_offset[1] if self.watermark is not None: # keep in image bounds, do this even when not dragging because the image sizes could change. self.pos[1] = max( 0, min(frame.img.shape[0] - self.watermark.shape[0], max(self.pos[1], 0)), ) self.pos[0] = max( 0, min(frame.img.shape[1] - self.watermark.shape[1], max(self.pos[0], 0)), ) pos = int(self.pos[0]), int(self.pos[1]) img = frame.img roi = ( slice(pos[1], pos[1] + self.watermark.shape[0]), slice(pos[0], pos[0] + self.watermark.shape[1]), ) w_roi = slice(0, img.shape[0] - pos[1]), slice(0, img.shape[1] - pos[0]) img[roi] = self.watermark[w_roi] * self.alpha_mask[w_roi] + img[roi] * ( 1 - self.alpha_mask[w_roi] )
def on_button(window,button, action, mods): g_pool.gui.update_button(button,action,mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action)
def on_click(self, img_pos, button, action): """ gets called when the user clicks in the window screen """ pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: screen_seek_pos = self.seek_bar_to_screen( (self.current_frame_index, 0)) dist = abs(pos[0] - screen_seek_pos[0]) + abs(pos[1] - screen_seek_pos[1]) if dist < 20: self.drag_mode = True self.was_playing = self.g_pool.play self.g_pool.play = False elif action == GLFW_RELEASE: if self.drag_mode: x, _ = self.screen_to_seek_bar(pos) x = int(min(self.frame_count - 5, max(0, x))) try: self.cap.seek_to_frame(x) except: pass self.g_pool.new_seek = True self.drag_mode = False self.g_pool.play = self.was_playing
def on_click(self, img_pos, button, action): """ gets called when the user clicks in the window screen """ pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: screen_in_mark_pos = self.bar_space_to_screen((self.in_mark, 0)) screen_out_mark_pos = self.bar_space_to_screen((self.out_mark, 0)) #in mark dist = abs(pos[0] - screen_in_mark_pos[0]) + abs(pos[1] - screen_in_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.in_mark, self.capture.get_frame_index()) > 20: self.drag_in = True return #out mark dist = abs(pos[0] - screen_out_mark_pos[0]) + abs(pos[1] - screen_out_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.out_mark, self.capture.get_frame_index()) > 20: self.drag_out = True elif action == GLFW_RELEASE: self.drag_out = False self.drag_in = False
def on_window_mouse_button(window, button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = normalize(pos, camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size + 40, g_pool.u_r.handle_size + 40): # if the roi interacts we dont want # the gui to interact as well return g_pool.gui.update_button(button, action, mods)
def uroi_on_mouse_button(button, action, mods): if g_pool.display_mode == "roi": if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False # if the roi interacts we dont want # the gui to interact as well return elif action == glfw.GLFW_PRESS: x, y = glfw.glfwGetCursorPos(main_window) # pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) x *= hdpi_factor y *= hdpi_factor pos = normalize((x, y), camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize( pos, g_pool.capture.frame_size ) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt( pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size ): # if the roi interacts we dont want # the gui to interact as well return
def on_click(self,img_pos,button,action): """ gets called when the user clicks in the window screen """ pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0)) screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0)) #in mark dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20: self.drag_in=True return #out mark dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20: self.drag_out=True elif action == GLFW_RELEASE: self.drag_out = False self.drag_in = False
def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) pos = glfw.glfwGetCursorPos(window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action)
def current_mouse_pos(window, camera_render_size, frame_size): hdpi_fac = getHDPIFactor(window) x, y = glfwGetCursorPos(glfwGetCurrentContext()) pos = x * hdpi_fac, y * hdpi_fac pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, frame_size) return (int(pos[0]), int(pos[1]))
def current_mouse_pos(window, camera_render_size, frame_size): content_scale = get_content_scale(window) x, y = glfwGetCursorPos(glfwGetCurrentContext()) pos = x * content_scale, y * content_scale pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, frame_size) return (int(pos[0]), int(pos[1]))
def recent_events(self, events): frame = events.get('frame') if not frame: return if frame.index == self.out_mark or frame.index == self.in_mark: self.g_pool.play = False if self.drag_in: x,y = glfwGetCursorPos(glfwGetCurrentContext()) x,_ = self.screen_to_bar_space((x,y)) self.in_mark = x elif self.drag_out: x,y = glfwGetCursorPos(glfwGetCurrentContext()) x,_ = self.screen_to_bar_space((x,y)) self.out_mark = x
def recent_events(self, events): frame = events.get('frame') if not frame: return if frame.index == self.out_mark or frame.index == self.in_mark: self.g_pool.play = False if self.drag_in: x, y = glfwGetCursorPos(glfwGetCurrentContext()) x, _ = self.screen_to_bar_space((x, y)) self.in_mark = x elif self.drag_out: x, y = glfwGetCursorPos(glfwGetCurrentContext()) x, _ = self.screen_to_bar_space((x, y)) self.out_mark = x
def update(self,frame,recent_pupil_positions,events): self.current_frame_index = frame.index self.norm_seek_pos = self.current_frame_index/float(self.frame_count) if self.drag_mode: pos = glfwGetCursorPos(glfwGetCurrentContext()) norm_seek_pos, _ = self.screen_to_seek_bar(pos) norm_seek_pos = min(1,max(0,norm_seek_pos)) if abs(norm_seek_pos-self.norm_seek_pos) >=.01: seek_pos = int(norm_seek_pos*self.frame_count) self.cap.seek_to_frame(seek_pos) self.g_pool.new_seek = True
def update(self, frame, recent_pupil_positions, events): self.current_frame_index = frame.index self.norm_seek_pos = self.current_frame_index / float(self.frame_count) if self.drag_mode: pos = glfwGetCursorPos(glfwGetCurrentContext()) norm_seek_pos, _ = self.screen_to_seek_bar(pos) norm_seek_pos = min(1, max(0, norm_seek_pos)) if abs(norm_seek_pos - self.norm_seek_pos) >= .01: seek_pos = int(norm_seek_pos * self.frame_count) self.cap.seek_to_frame(seek_pos) self.g_pool.new_seek = True
def mouseButtons_callback(window, button, action, mods): global V global x0,y0,w0,h0,b0state,b1state # select region if button==glfw.GLFW_MOUSE_BUTTON_RIGHT and action==glfw.GLFW_PRESS: x,y = glfw.glfwGetCursorPos (window) x0,y0 = V.compute_image_coordinates(x,y) w0,h0=0,0 b1state='pressed' V.redisp=1 elif button==glfw.GLFW_MOUSE_BUTTON_RIGHT and action==glfw.GLFW_RELEASE: x,y = glfw.glfwGetCursorPos (window) curr_x,curr_y = V.compute_image_coordinates(x,y) # print curr_x, curr_y w0,h0 = int(curr_x)-int(x0),int(curr_y)-int(y0) b1state='released' xx0,yy0 = x0,y0 xx1,yy1 = x0+w0,y0+h0 xx0,yy0,xx1,yy1 = int(xx0),int(yy0),int(xx1),int(yy1) print(xx0, yy0, abs(xx1-xx0), abs(yy1-yy0)) V.redisp=1 # drag if button==glfw.GLFW_MOUSE_BUTTON_LEFT and action==glfw.GLFW_PRESS: x,y = glfw.glfwGetCursorPos (window) V.dragx0,V.dragy0 = V.compute_image_coordinates(x,y) V.dragdx,V.dragdy=0,0 b0state='pressed' V.redisp=1 elif button==glfw.GLFW_MOUSE_BUTTON_LEFT and action==glfw.GLFW_RELEASE: x,y = glfw.glfwGetCursorPos (window) curr_x,curr_y = V.compute_image_coordinates(x,y) V.dragdx,V.dragdy = curr_x-V.dragx0,curr_y-V.dragy0 b0state='released' V.dx=V.dx-V.dragdx V.dy=V.dy-V.dragdy V.dragdx=0 V.dragdy=0 V.redisp=1
def consume_events_and_render_buffer(): glfw.glfwMakeContextCurrent(main_window) clear_gl_screen() if all(c > 0 for c in g_pool.camera_render_size): glViewport(0, 0, *g_pool.camera_render_size) for p in g_pool.plugins: p.gl_display() glViewport(0, 0, *window_size) # render graphs fps_graph.draw() cpu_graph.draw() # render GUI try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = glfw.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None ) pos = normalize(pos, g_pool.camera_render_size) if g_pool.flip: pos = 1 - pos[0], 1 - pos[1] # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # update screen glfw.glfwSwapBuffers(main_window)
def __mousefunc(self, window, button, action, mods): fw, fh = glfw.glfwGetFramebufferSize(window) ww, wh = glfw.glfwGetWindowSize(window) x, y = glfw.glfwGetCursorPos(window) sx = int(x * float(fw) / float(ww)) sy = int(y * float(fh) / float(wh)) if action == glfw.GLFW_PRESS: state = self.BUTTON_DOWN else: action = self.BUTTON_UP if button == glfw.GLFW_MOUSE_BUTTON_LEFT: button = self.BUTTON_LEFT elif button == glfw.GLFW_MOUSE_BUTTON_RIGHT: button = self.BUTTON_RIGHT else: return # Don't support middle button self.buttons[button] = action self.input2d(sx, sy, self.buttons)
def recent_events(self,events): if 'frame' in events: frame = events['frame'] self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray # hack "self.markers" instead "self.screens" is keeped for inheritence compatibility self.markers = detect_screens(gray) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) events['frame'] = frame # locate surfaces, map gaze for s in self.surfaces: s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d) if s.detected: s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen) else: s.gaze_on_srf =[] events['surface'] = [] for s in self.surfaces: if s.detected: events['surface'].append({ 'name':s.name, 'uid':s.uid, 'm_to_screen':s.m_to_screen.tolist(), 'm_from_screen':s.m_from_screen.tolist(), 'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp, 'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None }) if self.running: self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos)
def update(self,frame,events): self.current_frame_index = frame.index if self.drag_mode: x,y = glfwGetCursorPos(glfwGetCurrentContext()) x,_ = self.screen_to_seek_bar((x,y)) seek_pos = min(self.frame_count,max(0,x)) if abs(seek_pos-self.current_frame_index) >=.002*self.frame_count: seek_pos = int(min(seek_pos,self.frame_count-5)) #the last frames can be problematic to seek to try: self.cap.seek_to_frame(seek_pos) self.current_frame_index = seek_pos except: pass self.g_pool.new_seek = True
def on_button(window,button, action, mods): if g_pool.display_mode == 'roi': if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt: g_pool.u_r.active_edit_pt = False return # if the roi interacts we dont what the gui to interact as well elif action == glfw.GLFW_PRESS: pos = glfw.glfwGetCursorPos(window) pos = normalize(pos,glfw.glfwGetWindowSize(main_window)) if g_pool.flip: pos = 1-pos[0],1-pos[1] pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40): return # if the roi interacts we dont what the gui to interact as well g_pool.gui.update_button(button,action,mods)
def update(self,frame,events): self.current_frame_index = frame.index if self.drag_mode: x,y = glfwGetCursorPos(glfwGetCurrentContext()) x,_ = self.screen_to_seek_bar((x,y)) seek_pos = min(self.frame_count,max(0,x)) seek_pos = int(min(seek_pos,self.frame_count-5)) #the last frames can be problematic to seek to if self.current_frame_index-1 != seek_pos: try: # logger.info('seeking to {} form {}'.format(seek_pos,self.current_frame_index)) self.cap.seek_to_frame(seek_pos) self.current_frame_index = self.cap.get_frame_index() except: pass self.g_pool.new_seek = True
def update(self,frame,events): if self.drag_offset is not None: pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels self.pos[0] = pos[0]+self.drag_offset[0] self.pos[1] = pos[1]+self.drag_offset[1] if self.watermark is not None: #keep in image bounds, do this even when not dragging because the image sizes could change. self.pos[1] = min(frame.img.shape[0]-self.watermark.shape[0],max(self.pos[1],0)) self.pos[0] = min(frame.img.shape[1]-self.watermark.shape[1],max(self.pos[0],0)) pos = int(self.pos[0]),int(self.pos[1]) img = frame.img roi = slice(pos[1],pos[1]+self.watermark.shape[0]),slice(pos[0],pos[0]+self.watermark.shape[1]) img[roi] = self.watermark*self.alpha_mask + img[roi]*(1-self.alpha_mask)
def update(self,frame,events): for eye_index in self.showeyes: requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index] #1. do we need a new frame? if requested_eye_frame_idx != self.eye_frames[eye_index].index: # do we need to seek? if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1: # if we just need to seek by one frame, its faster to just read one and and throw it away. _ = self.eye_cap[eye_index].get_frame() if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index(): # only now do I need to seek self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx) # reading the new eye frame frame try: self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame() except EndofVideoFileError: logger.warning("Reached the end of the eye video for eye video %s."%eye_index) else: #our old frame is still valid because we are doing upsampling pass #2. dragging image if self.drag_offset[eye_index] is not None: pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0] self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1] else: self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)] #3. keep in image bounds, do this even when not dragging because the image video_sizes could change. self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0)) #4. flipping images, converting to greyscale eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling eyeimage = cv2.resize(eye_gray,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor) if self.mirror[str(eye_index)]: eyeimage = np.fliplr(eyeimage) if self.flip[str(eye_index)]: eyeimage = np.flipud(eyeimage) #5. finally overlay the image x,y = int(self.pos[eye_index][0]),int(self.pos[eye_index][1]) transparent_image_overlay((x,y),cv2.cvtColor(eyeimage,cv2.COLOR_GRAY2BGR),frame.img,self.alpha)
def update(self, frame, events): self.current_frame_index = frame.index if self.drag_mode: x, y = glfwGetCursorPos(glfwGetCurrentContext()) x, _ = self.screen_to_seek_bar((x, y)) seek_pos = min(self.frame_count, max(0, x)) seek_pos = int( min(seek_pos, self.frame_count - 5)) #the last frames can be problematic to seek to if self.current_frame_index - 1 != seek_pos: try: # logger.info('seeking to {} form {}'.format(seek_pos,self.current_frame_index)) self.cap.seek_to_frame(seek_pos) self.current_frame_index = self.cap.get_frame_index() except: pass self.g_pool.new_seek = True
def update(self, frame, recent_pupil_positions, events): self.current_frame_index = frame.index if self.drag_mode: x, y = glfwGetCursorPos(glfwGetCurrentContext()) x, _ = self.screen_to_seek_bar((x, y)) seek_pos = min(self.frame_count, max(0, x)) if abs(seek_pos - self.current_frame_index) >= .002 * self.frame_count: seek_pos = int( min(seek_pos, self.frame_count - 5)) #the last frames can be problematic to seek to try: self.cap.seek_to_frame(seek_pos) self.current_frame_index = seek_pos except: pass self.g_pool.new_seek = True
def update(self,frame,events): self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray # hack "self.markers" instead "self.screens" is keeped for inheritence compatibility self.markers = detect_screens(gray) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) events['surfaces'] = [] # locate surfaces for s in self.surfaces: s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d) if s.detected: events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.running: self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze_positions',[]): gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s)
def consume_events_and_render_buffer(): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = glfw.window_coordinate_to_framebuffer_coordinate( main_window, x, y, cached_scale=None) pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break glfw.glfwSwapBuffers(main_window)
def on_click(self,img_pos,button,action): """ gets called when the user clicks in the window screen """ hdpi_factor = float(glfwGetFramebufferSize(glfwGetCurrentContext())[0]/glfwGetWindowSize(glfwGetCurrentContext())[0]) pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = pos[0]*hdpi_factor,pos[1]*hdpi_factor #drag the seek point if action == GLFW_PRESS: screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0)) screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0)) #in mark dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20: self.drag_in=True return #out mark dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1]) if dist < 10: if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20: self.drag_out=True elif action == GLFW_RELEASE: if self.drag_out or self.drag_in: logger.info("Section: "+self.get_string()) self.drag_out = False self.drag_in = False # would be great to expand the click area horizontally for big sections for s in self.sections: if s is not self.sections[self.focus]: midsec = self.mid_sections[self.sections.index(s)] screen_midsec_pos = self.bar_space_to_screen((midsec,0)) dist = abs(pos[0]-screen_midsec_pos[0])+abs(pos[1]-screen_midsec_pos[1]) if dist < 10: if self.distance_in_pix(midsec,self.capture.get_frame_index()) > 20: self.focus = self.sections.index(s) break
def update(self, frame, events): if self.drag_offset is not None: pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos, glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels self.pos[0] = pos[0] + self.drag_offset[0] self.pos[1] = pos[1] + self.drag_offset[1] if self.watermark is not None: #keep in image bounds, do this even when not dragging because the image sizes could change. self.pos[1] = min(frame.img.shape[0] - self.watermark.shape[0], max(self.pos[1], 0)) self.pos[0] = min(frame.img.shape[1] - self.watermark.shape[1], max(self.pos[0], 0)) pos = int(self.pos[0]), int(self.pos[1]) img = frame.img roi = slice(pos[1], pos[1] + self.watermark.shape[0]), slice( pos[0], pos[0] + self.watermark.shape[1]) img[roi] = self.watermark * self.alpha_mask + img[roi] * ( 1 - self.alpha_mask)
def on_click(self,img_pos,button,action): """ gets called when the user clicks in the window screen """ pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: screen_seek_pos = self.seek_bar_to_screen((self.norm_seek_pos,0)) dist = abs(pos[0]-screen_seek_pos[0])+abs(pos[1]-screen_seek_pos[1]) if dist < 20: self.drag_mode=True self.was_playing = self.g_pool.play self.g_pool.play = False elif action == GLFW_RELEASE: if self.drag_mode: norm_seek_pos, _ = self.screen_to_seek_bar(pos) norm_seek_pos = min(1,max(0,norm_seek_pos)) seek_pos = int(norm_seek_pos*self.frame_count) self.cap.seek_to_frame(seek_pos) self.g_pool.new_seek = True self.drag_mode=False self.g_pool.play = self.was_playing
def mouseWheel_callback(window, xoffset, yoffset): global V,D if V.mute_wheel: V.mute_wheel_buffer[0]=V.mute_wheel_buffer[0]+xoffset V.mute_wheel_buffer[1]=V.mute_wheel_buffer[1]+yoffset return else: xoffset=xoffset+V.mute_wheel_buffer[0] yoffset=yoffset+V.mute_wheel_buffer[1] V.mute_wheel_buffer = [0,0] V.mute_wheel=1 curr_x,curr_y = glfw.glfwGetCursorPos (window) # zoom if V.alt_is_pressed: V.zoom_update(yoffset*GLOBAL_WHEEL_SCALING,curr_x,curr_y) # scale elif V.shift_is_pressed: V.radius_update(yoffset*GLOBAL_WHEEL_SCALING) # bias else: # nothing pressed V.center_update(yoffset*GLOBAL_WHEEL_SCALING)
def on_click(self, img_pos, button, action): """ gets called when the user clicks in the window screen """ pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: screen_seek_pos = self.seek_bar_to_screen((self.norm_seek_pos, 0)) dist = abs(pos[0] - screen_seek_pos[0]) + abs(pos[1] - screen_seek_pos[1]) if dist < 20: self.drag_mode = True self.was_playing = self.g_pool.play self.g_pool.play = False elif action == GLFW_RELEASE: if self.drag_mode: norm_seek_pos, _ = self.screen_to_seek_bar(pos) norm_seek_pos = min(1, max(0, norm_seek_pos)) seek_pos = int(norm_seek_pos * self.frame_count) self.cap.seek_to_frame(seek_pos) self.g_pool.new_seek = True self.drag_mode = False self.g_pool.play = self.was_playing
def world( timebase, eye_procs_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version, preferred_remote_port, ): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { "subject": "eye_process.should_start.{}".format(eye_id), "eye_id": eye_id, "delay": delay, } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { "subject": "eye_process.should_stop.{}".format(eye_id), "eye_id": eye_id, "delay": 0.2, } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {"subject": "set_detection_mapping_mode", "mode": new_mode} ipc_pub.notify(n) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( "1.23" ), "pyglui out of date, please upgrade to newest version" from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info("Application Version: {}".format(version)) logger.info("System Info: {}".format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import ( Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins, ) from plugin_manager import Plugin_Manager from calibration_routines import ( calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin, ) from fixation_detector import Fixation_Detector from eye_movement import Eye_Movement_Detector_Real_Time from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker_Online from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import ( source_classes, manager_classes, Base_Manager, Base_Source, ) from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay # UI Platform tweaks if platform.system() == "Linux": scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == "Windows": scroll_factor = 10.0 window_position_default = (8, 90) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() g_pool.app = "capture" g_pool.process = "world" g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eye_procs_alive = eye_procs_alive g_pool.preferred_remote_port = preferred_remote_port def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins") ) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker_Online, Annotation_Capture, Log_History, Fixation_Detector, Eye_Movement_Detector_Real_Time, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay, ] if platform.system() != "Windows": # Head pose tracking is currently not available on Windows from head_pose_tracker.online_head_pose_tracker import ( Online_Head_Pose_Tracker, ) user_plugins.append(Online_Head_Pose_Tracker) system_plugins = ( [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs, ] + manager_classes + source_classes ) plugins = ( system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins ) user_plugins += [ p for p in runtime_plugins if not isinstance( p, ( Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin, ), ) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e", ], "frame_size": (1280, 720), "frame_rate": 30, } default_plugins = [ ("UVC_Source", default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), ("Screen_Marker_Calibration", {}), ("Recorder", {}), ("Pupil_Remote", {}), ("Accuracy_Visualizer", {}), ("Plugin_Manager", {}), ("System_Graphs", {}), ] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in g_pool.plugins) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, "user_settings_world") ) if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", 0.8 ) g_pool.detection_mapping_mode = session_settings.get( "detection_mapping_mode", "3d" ) g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get("audio_mode", audio.default_audio_mode) def handle_notifications(noti): subject = noti["subject"] if subject == "set_detection_mapping_mode": if noti["mode"] == "2d": if ( "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name ): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add(g_pool.plugin_by_name["Dummy_Gaze_Mapper"]) g_pool.detection_mapping_mode = noti["mode"] elif subject == "start_plugin": g_pool.plugins.add( g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {}) ) elif subject == "stop_plugin": for p in g_pool.plugins: if p.class_name == noti["name"]: p.alive = False g_pool.plugins.clean() elif subject == "eye_process.started": noti = { "subject": "set_detection_mapping_mode", "mode": g_pool.detection_mapping_mode, } ipc_pub.notify(noti) elif subject == "set_min_calibration_confidence": g_pool.min_calibration_confidence = noti["value"] elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) width, height = session_settings.get( "window_size", (1280 + icon_bar_width, 720) ) # window and gl setup glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get("window_position", window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0}) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is opened, the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.6, 0.8, 1.0, 1.2, 1.4], label="Interface size", ) ) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector("audio_mode", audio, selection=audio.audio_modes) ) general_settings.append( ui.Selector( "detection_mapping_mode", g_pool, label="detection & mapping mode", setter=set_detection_mapping_mode, selection=["disabled", "2d", "3d"], ) ) general_settings.append( ui.Switch( "eye0_process", label="Detect eye 0", setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eye_procs_alive[0].value, ) ) general_settings.append( ui.Switch( "eye1_process", label="Detect eye 1", setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eye_procs_alive[1].value, ) ) general_settings.append( ui.Info_Text("Capture Version: {}".format(g_pool.version)) ) general_settings.append( ui.Button("Restart with default settings", reset_restart) ) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins) ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(True) # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get("eye1_process_alive", True): launch_eye_process(1, delay=0.6) if session_settings.get("eye0_process_alive", True): launch_eye_process(0, delay=0.3) ipc_pub.notify({"subject": "world_process.started"}) logger.warning("Process started.") # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events["dt"] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # "blacklisted" events that were already sent del events["pupil"] del events["gaze"] # delete if exists. More expensive than del, so only use it when key might not exist events.pop("annotation", None) # send new events to ipc: if "frame" in events: del events["frame"] # send explicitly with frame publisher if "depth_frame" in events: del events["depth_frame"] if "audio_packets" in events: del events["audio_packets"] del events["dt"] # no need to send this for data in events.values(): assert isinstance(data, (list, tuple)) for d in data: ipc_pub.send(d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipboard is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString( main_window, user_input.clipboard.encode() ) for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) # call `on_click` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_click(pos, button, action) for p in g_pool.plugins) for key, scancode, action, mods in user_input.keys: # call `on_key` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_key(key, scancode, action, mods) for p in g_pool.plugins) for char_ in user_input.chars: # call `char_` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_char(char_) for p in g_pool.plugins) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value session_settings["eye1_process_alive"] = eye_procs_alive[1].value session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["detection_mapping_mode"] = g_pool.detection_mapping_mode session_settings["audio_mode"] = audio.audio_mode session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error("Process Capture crashed with trace:\n{}".format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({"subject": "world_process.stopped"}) sleep(1.0)
def visualize(self, frame, alpha, scale, show_ellipses, pupil_positions): if not self.initialized: return requested_eye_frame_idx = self.eye_world_frame_map[frame.index] # 1. do we need a new frame? if requested_eye_frame_idx != self.current_eye_frame.index: if requested_eye_frame_idx == self.source.get_frame_index() + 2: # if we just need to seek by one frame, its faster to just read one and and throw it away. self.source.get_frame() if requested_eye_frame_idx != self.source.get_frame_index() + 1: self.source.seek_to_frame(int(requested_eye_frame_idx)) try: self.current_eye_frame = self.source.get_frame() except EndofVideoError: logger.info( "Reached the end of the eye video for eye video {}.". format(self.eyeid)) # 2. dragging image if self.drag_offset is not None: x, y = glfwGetCursorPos(glfwGetCurrentContext()) pos = x * self.hdpi_fac, y * self.hdpi_fac pos = normalize(pos, self.g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) self.pos = ( int(pos[0] + self.drag_offset[0]), int(pos[1] + self.drag_offset[1]), ) # 3. keep in image bounds, do this even when not dragging because the image video_sizes could change. video_size = ( round(self.current_eye_frame.width * scale), round(self.current_eye_frame.height * scale), ) # frame.img.shape[0] is height, frame.img.shape[1] is width of screen self.pos = ( min(frame.img.shape[1] - video_size[0], max(self.pos[0], 0)), min(frame.img.shape[0] - video_size[1], max(self.pos[1], 0)), ) # 4. flipping images, converting to greyscale eyeimage = self.current_eye_frame.gray eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR) if show_ellipses: try: pp = next( (pp for pp in pupil_positions if pp["id"] == self.eyeid and pp["timestamp"] == self.current_eye_frame.timestamp)) except StopIteration: pass else: draw_pupil_on_image(eyeimage, pp) # flip and scale eyeimage = cv2.resize(eyeimage, (0, 0), fx=scale, fy=scale) if self.hflip: eyeimage = np.fliplr(eyeimage) if self.vflip: eyeimage = np.flipud(eyeimage) transparent_image_overlay(self.pos, eyeimage, frame.img, alpha)
def gl_display(self): from math import floor if self.depth_window is not None and glfw.glfwWindowShouldClose( self.depth_window ): glfw.glfwDestroyWindow(self.depth_window) self.depth_window = None if self.depth_window is not None and self._recent_depth_frame is not None: active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(self.depth_window) win_size = glfw.glfwGetFramebufferSize(self.depth_window) gl_utils.adjust_gl_view(win_size[0], win_size[1]) pos = glfw.glfwGetCursorPos(self.depth_window) if self.mouse_drag: self.pitch = np.clip(self.pitch + (pos[1] - self.last_pos[1]), -80, 80) self.yaw = np.clip(self.yaw - (pos[0] - self.last_pos[0]), -120, 120) self.last_pos = pos glClearColor(0, 0, 0, 0) glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) glMatrixMode(GL_PROJECTION) glLoadIdentity() gluPerspective(60, win_size[0] / win_size[1], 0.01, 20.0) glMatrixMode(GL_MODELVIEW) glLoadIdentity() gluLookAt(0, 0, 0, 0, 0, 1, 0, -1, 0) glTranslatef(0, 0, 0.5) glRotated(self.pitch, 1, 0, 0) glRotated(self.yaw, 0, 1, 0) glTranslatef(0, 0, -0.5) # glPointSize(2) glEnable(GL_DEPTH_TEST) extrinsics = self.device.get_device_extrinsics( rs_stream.RS_STREAM_DEPTH, rs_stream.RS_STREAM_COLOR ) depth_frame = self._recent_depth_frame color_frame = self._recent_frame depth_scale = self.device.depth_scale glEnableClientState(GL_VERTEX_ARRAY) pointcloud = self.device.pointcloud glVertexPointer(3, GL_FLOAT, 0, pointcloud) glEnableClientState(GL_COLOR_ARRAY) depth_to_color = np.zeros( depth_frame.height * depth_frame.width * 3, np.uint8 ) rsutilwrapper.project_pointcloud_to_pixel( depth_to_color, self.device.depth_intrinsics, self.device.color_intrinsics, extrinsics, pointcloud, self._recent_frame.bgr, ) glColorPointer(3, GL_UNSIGNED_BYTE, 0, depth_to_color) glDrawArrays(GL_POINTS, 0, depth_frame.width * depth_frame.height) gl_utils.glFlush() glDisable(GL_DEPTH_TEST) # gl_utils.make_coord_system_norm_based() glfw.glfwSwapBuffers(self.depth_window) glfw.glfwMakeContextCurrent(active_window) if self.preview_depth and self._recent_depth_frame is not None: self.g_pool.image_tex.update_from_ndarray(self._recent_depth_frame.bgr) gl_utils.glFlush() gl_utils.make_coord_system_norm_based() self.g_pool.image_tex.draw() elif self._recent_frame is not None: self.g_pool.image_tex.update_from_yuv_buffer( self._recent_frame.yuv_buffer, self._recent_frame.width, self._recent_frame.height, ) gl_utils.glFlush() gl_utils.make_coord_system_norm_based() self.g_pool.image_tex.draw() if not self.online: super().gl_display() gl_utils.make_coord_system_pixel_based( (self.frame_size[1], self.frame_size[0], 3) )
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging import errno from glob import glob from time import time # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source, EndofVideoFileError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from batch_exporter import Batch_Exporter, Batch_Export from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection assert VersionFormat(pyglui_version) >= VersionFormat( '1.17'), 'pyglui out of date, please upgrade to newest version' runtime_plugins = import_runtime_plugins( os.path.join(user_dir, 'plugins')) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export, System_Timelines ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug( "Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": new_rec_dir }) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format( new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg') ][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None # sets itself to g_pool.capture File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() g_pool.capture.playback_speed = session_settings.get( 'playback_speed', 1.) width, height = session_settings.get('window_size', g_pool.capture.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( 'min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data( g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer g_pool.fixations_by_frame = [ [] for x in g_pool.timestamps ] # populated by the fixation detector plugin def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time() + .8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = { 'subject': 'should_export', 'range': export_range, 'export_dir': export_dir } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.), size=(0., 0.), header_pos='headline') g_pool.user_timelines.color = RGBA(a=0.) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, g_pool.capture.frame_size[0], g_pool.capture. frame_size[1]))) general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2] + list(np.arange(1.5, 5.1, .5)), label='Interface Size')) general_settings.append( ui.Info_Text('Player Version: {}'.format(g_pool.version))) general_settings.append( ui.Info_Text('Capture Version: {}'.format( meta_info['Capture Software Version']))) general_settings.append( ui.Info_Text('Data Format Version: {}'.format( meta_info['Data Format Version']))) general_settings.append( ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100)) g_pool.export_button = ui.Thumb('export', label=chr(0xe2c5), getter=lambda: False, setter=do_export, hotkey='e', label_font='pupil_icons') g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}), ('Raw_Data_Exporter', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}), ('System_Timelines', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.capture.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = g_pool.capture.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.capture.play = False logger.warning("end of video") frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # pupil and gaze positions are added by their respective producer plugins events['pupil_positions'] = [] events['gaze_positions'] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture._recent_frame = frame g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = '' g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) # present frames at appropriate speed g_pool.capture.wait(frame) glfw.glfwPollEvents() session_settings['playback_speed'] = g_pool.capture.playback_speed session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.capture.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error('Process Player crashed with trace:\n{}'.format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'}) sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version, debug): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify", )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from pupil_recording import PupilRecording from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from seek_control import Seek_Control from surface_tracker import Surface_Tracker_Offline # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay from pupil_recording import ( assert_valid_recording_type, InvalidRecordingException, ) assert VersionFormat(pyglui_version) >= VersionFormat( "1.27"), "pyglui out of date, please upgrade to newest version" process_was_interrupted = False def interrupt_handler(sig, frame): import traceback trace = traceback.format_stack(f=frame) logger.debug(f"Caught signal {sig} in:\n" + "".join(trace)) nonlocal process_was_interrupted process_was_interrupted = True signal.signal(signal.SIGINT, interrupt_handler) runtime_plugins = import_runtime_plugins( os.path.join(user_dir, "plugins")) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, Offline_Fixation_Detector, Offline_Blink_Detection, Surface_Tracker_Offline, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, Offline_Head_Pose_Tracker, ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for path in paths: try: assert_valid_recording_type(path) _restart_with_recording(path) return except InvalidRecordingException as err: logger.debug(str(err)) for plugin in g_pool.plugins: if plugin.on_drop(paths): break def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": rec_dir }) glfw.glfwSetWindowShouldClose(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) recording = PupilRecording(rec_dir) meta_info = recording.meta_info # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) logger.debug(f"Debug flag: {debug}") icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() g_pool.app = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None video_path = recording.files().core().world().videos()[0].resolve() File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_pos = session_settings.get("window_position", window_position_default) window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}" glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( "min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT) # populated by producers g_pool.pupil_positions = pm.PupilDataBisector() g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() g_pool.eye_movements = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info.min_player_version, "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify({ "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left") g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden") g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu("User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline") g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), label="Interface Size", )) general_settings.append( ui.Info_Text( f"Minimum Player Version: {meta_info.min_player_version}")) general_settings.append( ui.Info_Text(f"Player Version: {g_pool.version}")) general_settings.append( ui.Info_Text( f"Recording Software: {meta_info.recording_software_name}")) general_settings.append( ui.Info_Text( f"Recording Software Version: {meta_info.recording_software_version}" )) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." )) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", )) general_settings.append( ui.Button("Restart with default settings", reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey="e", label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), ("Pupil_From_Recording", {}), ("GazeFromRecording", {}), ("Audio_Playback", {}), ] g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins)) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add(g_pool.plugin_by_name[n["name"]], args=n.get("args", {})) elif subject.startswith("meta.should_doc"): ipc_pub.notify({ "subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, }) while (not glfw.glfwWindowShouldClose(main_window) and not process_was_interrupted): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) glfw.glfwPollEvents() # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for plugin in g_pool.plugins: if plugin.on_click(pos, button, action): break for key, scancode, action, mods in user_input.keys: for plugin in g_pool.plugins: if plugin.on_key(key, scancode, action, mods): break for char_ in user_input.chars: for plugin in g_pool.plugins: if plugin.on_char(char_): break # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.glfwSwapBuffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence"] = g_pool.min_calibration_confidence session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos( main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except Exception: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging from glob import glob from time import time, strftime, localtime # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.NOTSET) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: from background_helper import IPC_Logging_Task_Proxy IPC_Logging_Task_Proxy.push_url = ipc_push_url from tasklib.background.patches import IPCLoggingPatch IPCLoggingPatch.ipc_push_url = ipc_push_url # imports from file_methods import Persistent_Dict, next_export_sub_dir # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info import player_methods as pm from csv_utils import write_key_value_file # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation # from vis_scan_path import Vis_Scan_Path from seek_control import Seek_Control from surface_tracker import Surface_Tracker_Offline # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from eye_movement import Offline_Eye_Movement_Detector from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producer.gaze_from_recording import GazeFromRecording from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection from audio_playback import Audio_Playback from video_export.plugins.imotions_exporter import iMotions_Exporter from video_export.plugins.eye_video_exporter import Eye_Video_Exporter from video_export.plugins.world_video_exporter import World_Video_Exporter from video_capture import File_Source from video_overlay.plugins import Video_Overlay, Eye_Overlay assert VersionFormat(pyglui_version) >= VersionFormat( "1.23" ), "pyglui out of date, please upgrade to newest version" runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins")) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, System_Timelines, Audio_Playback, ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Eye_Overlay, Video_Overlay, # Vis_Scan_Path, Offline_Fixation_Detector, Offline_Eye_Movement_Detector, Offline_Blink_Detection, Surface_Tracker_Offline, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, GazeFromRecording, GazeFromOfflineCalibration, World_Video_Exporter, iMotions_Exporter, Eye_Video_Exporter, ] + runtime_plugins if platform.system() != "Windows": # Head pose tracking is currently not available on Windows from head_pose_tracker.offline_head_pose_tracker import ( Offline_Head_Pose_Tracker, ) user_plugins.append(Offline_Head_Pose_Tracker) plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor if w == 0 or h == 0: return hdpi_factor = glfw.getHDPIFactor(window) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode("utf-8") for x in range(count)] for path in paths: if pm.is_pupil_rec_dir(path): _restart_with_recording(path) return # call `on_drop` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_drop(paths) for p in g_pool.plugins) def _restart_with_recording(rec_dir): logger.debug("Starting new session with '{}'".format(rec_dir)) ipc_pub.notify( {"subject": "player_drop_process.should_start", "rec_dir": rec_dir} ) glfw.glfwSetWindowShouldClose(g_pool.main_window, True) tick = delta_t() def get_dt(): return next(tick) meta_info = pm.load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info("Application Version: {}".format(app_version)) logger.info("System Info: {}".format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() g_pool.app = "player" g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake") video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in valid_ext ][0] File_Source( g_pool, timing="external", source_path=video_path, buffered_decoding=True, fill_gaps=True, ) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player") ) if VersionFormat(session_settings.get("version", "0.0")) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() width, height = g_pool.capture.frame_size width += icon_bar_width width, height = session_settings.get("window_size", (width, height)) window_pos = session_settings.get("window_position", window_position_default) window_name = "Pupil Player: {} - {}".format( meta_info["Recording Name"], os.path.split(rec_dir)[-1] ) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1], ) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.binocular = meta_info.get("Eye Mode", "monocular") == "binocular" g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get("min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT) g_pool.min_calibration_confidence = session_settings.get( "min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT ) # populated by producers g_pool.pupil_positions = pm.Bisector() g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector()) g_pool.gaze_positions = pm.Bisector() g_pool.fixations = pm.Affiliator() g_pool.eye_movements = pm.Affiliator() def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {"subject": "min_data_confidence_changed"} notification["_notify_time_"] = time() + 0.8 g_pool.ipc_pub.notify(notification) def do_export(_): left_idx = g_pool.seek_control.trim_left right_idx = g_pool.seek_control.trim_right export_range = left_idx, right_idx + 1 # exclusive range.stop export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx)) export_dir = os.path.join(g_pool.rec_dir, "exports") export_dir = next_export_sub_dir(export_dir) os.makedirs(export_dir) logger.info('Created export dir at "{}"'.format(export_dir)) export_info = { "Player Software Version": str(g_pool.version), "Data Format Version": meta_info["Data Format Version"], "Export Date": strftime("%d.%m.%Y", localtime()), "Export Time": strftime("%H:%M:%S", localtime()), "Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(), "Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(), "Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(), } with open(os.path.join(export_dir, "export_info.csv"), "w") as csv: write_key_value_file(csv, export_info) notification = { "subject": "should_export", "range": export_range, "ts_window": export_ts_window, "export_dir": export_dir, } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({"subject": "clear_settings_process.should_start"}) ipc_pub.notify( { "subject": "player_process.should_start", "rec_dir": rec_dir, "delay": 2.0, } ) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) g_pool.iconbar = ui.Scrolling_Menu( "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden" ) g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu( "User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline" ) g_pool.user_timelines.color = RGBA(a=0.0) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( ui.Selector( "gui_user_scale", g_pool, setter=set_scale, selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), label="Interface Size", ) ) general_settings.append( ui.Info_Text("Player Version: {}".format(g_pool.version)) ) general_settings.append( ui.Info_Text( "Capture Version: {}".format(meta_info["Capture Software Version"]) ) ) general_settings.append( ui.Info_Text( "Data Format Version: {}".format(meta_info["Data Format Version"]) ) ) general_settings.append( ui.Info_Text( "High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence." ) ) general_settings.append( ui.Slider( "min_data_confidence", g_pool, setter=set_data_confidence, step=0.05, min=0.0, max=1.0, label="Minimum data confidence", ) ) general_settings.append( ui.Button("Restart with default settings", reset_restart) ) g_pool.menubar.append(general_settings) icon = ui.Icon( "collapsed", general_settings, label=chr(0xE8B8), on_val=False, off_val=True, setter=toggle_general_settings, label_font="pupil_icons", ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100)) g_pool.export_button = ui.Thumb( "export", label=chr(0xE2C5), getter=lambda: False, setter=do_export, hotkey="e", label_font="pupil_icons", ) g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), ("Log_Display", {}), ("Raw_Data_Exporter", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), ("Pupil_From_Recording", {}), ("GazeFromRecording", {}), ("Audio_Playback", {}), ] g_pool.plugins = Plugin_List( g_pool, session_settings.get("loaded_plugins", default_plugins) ) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) g_pool.plugins._plugins.sort(key=lambda p: p.order) g_pool.capture.init_ui() general_settings.insert( -1, ui.Text_Input( "rel_time_trim_section", getter=g_pool.seek_control.get_rel_time_trim_range_string, setter=g_pool.seek_control.set_rel_time_trim_range_string, label="Relative time range to export", ), ) general_settings.insert( -1, ui.Text_Input( "frame_idx_trim_section", getter=g_pool.seek_control.get_frame_index_trim_range_string, setter=g_pool.seek_control.set_frame_index_trim_range_string, label="Frame index range to export", ), ) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n["subject"] if subject == "start_plugin": g_pool.plugins.add( g_pool.plugin_by_name[n["name"]], args=n.get("args", {}) ) elif subject.startswith("meta.should_doc"): ipc_pub.notify( {"subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__} ) for p in g_pool.plugins: if ( p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify ): ipc_pub.notify( { "subject": "meta.doc", "actor": p.class_name, "doc": p.on_notify.__doc__, } ) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) events = {} # report time between now and the last loop interation events["dt"] = get_dt() # pupil and gaze positions are added by their respective producer plugins events["pupil"] = [] events["gaze"] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString(main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = "" g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString( main_window, user_input.clipboard.encode() ) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) # call `on_click` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_click(pos, button, action) for p in g_pool.plugins) for key, scancode, action, mods in user_input.keys: # call `on_key` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_key(key, scancode, action, mods) for p in g_pool.plugins) for char_ in user_input.chars: # call `char_` callbacks until a plugin indicates # that it has consumed the event (by returning True) any(p.on_char(char_) for p in g_pool.plugins) # present frames at appropriate speed g_pool.seek_control.wait(events["frame"].timestamp) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() session_settings["min_data_confidence"] = g_pool.min_data_confidence session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: session_settings["window_size"] = session_window_size session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error("Process Player crashed with trace:\n{}".format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({"subject": "player_process.stopped"}) sleep(1.0)
def update(self, frame, events): for eye_index in self.showeyes: requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index] #1. do we need a new frame? if (requested_eye_frame_idx != self.eye_frames[eye_index].index or self.urActive) and self.recalculating == 0: # do we need to seek? if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1: # if we just need to seek by one frame, its faster to just read one and and throw it away. _ = self.eye_cap[eye_index].get_frame() if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index(): # only now do I need to seek self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx) # reading the new eye frame frame try: self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame() except EndofVideoFileError: logger.warning("Reached the end of the eye video for eye video {}.".format(eye_index)) else: #our old frame is still valid because we are doing upsampling pass #2. dragging image if self.drag_offset[eye_index] is not None: pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0] self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1] else: self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)] pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) pos = (int(pos[0] - self.pos[eye_index][0]), int(pos[1] - self.pos[eye_index][1])) self.mouse_img[eye_index] = pos if self.urActive == eye_index + 1: self.u_r[eye_index].move_vertex(self.u_r[eye_index].active_pt_idx,pos) if self.recalculating == 0: self.setPupilDetectors() pupil_detector = self.pupil_detectors[eye_index] glint_detector = self.glint_detectors[eye_index] settings = pupil_detector.get_settings() glint_settings = glint_detector.settings() self.setSettings(eye_index, settings, glint_settings) glint_detector.update() new_frame = self.eye_frames[eye_index] if self.algorithm == 1: view = "algorithm" else: view = False result, roi = pupil_detector.detect(new_frame, self.u_r[eye_index], view) glints = [[0,0,0,0,0,0], [0,0,0,0,0,1]] #glint_detector.glint(new_frame, eye_index, u_roi=self.u_r, pupil=result, roi=roi) if eye_index == 0: self.gPool0.pupil_queue.put(result) else: self.gPool1.pupil_queue.put(result) #3. keep in image bounds, do this even when not dragging because the image video_sizes could change. #self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen #self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0)) #4. flipping images, converting to greyscale #eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)), int(result['ellipse']['angle']),0,360,15) cv2.polylines(self.eye_frames[eye_index].img, [pts], 1, (0,255,0)) center = result['ellipse']['center'] center = [int(x) for x in center] cv2.circle(self.eye_frames[eye_index].img, tuple(center), True, (0,255,0), thickness=15) #img = 255 - np.mean(new_frame.img, axis=2) #img = (img*255).astype(np.uint8) #ret = pyelse.run(img, 10) #x = ret.center.x #y = ret.center.y #cv2.circle(self.eye_frames[eye_index].img, (int(x), int(y)), True, (255,0,0), thickness=10) glints = np.array(glints) #if len(glints)>0 and glints[0][3]: # for g in glints: # cv2.circle(self.eye_frames[eye_index].img, (int(g[1]),int(g[2])), True,(255,0,0),thickness=5) if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: cv2.polylines(self.eye_frames[eye_index].img, [pts], 1, (255,0,0)) if self.show_ellipses and events['pupil_positions'] and self.recalculating == 0: for pd in events['pupil_positions']: if pd['id'] == eye_index and pd['timestamp'] == self.eye_frames[eye_index].timestamp: break if pd['method'] == '3d c++': eye_ball = pd['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: cv2.polylines(self.eye_frames[eye_index].img, [pts], 1, (0,255,0)) el = pd['ellipse'] conf = int(pd.get('model_confidence', pd.get('confidence', 0.1)) * 255) center = list(map(lambda val: int(val), el['center'])) el['axes'] = tuple(map(lambda val: int(val/2), el['axes'])) el['angle'] = int(el['angle']) el_points = cv2.ellipse2Poly(tuple(center), el['axes'], el['angle'], 0, 360, 1) cv2.polylines(self.eye_frames[eye_index].img, [np.asarray(el_points)], True, (0, 0, 255, conf), thickness=math.ceil(2)) cv2.circle(self.eye_frames[eye_index].img, tuple(center), int(5*self.eye_scale_factor), (0, 0, 255, conf), thickness=-1) rect = np.asarray(self.u_r[eye_index].rect) #rect[:,[0, 1]] = rect[:,[1, 0]] cv2.polylines(self.eye_frames[eye_index].img, [rect],True, (0, 255, 255), thickness=math.ceil(2)) for corner in rect: cv2.circle(self.eye_frames[eye_index].img, (int(corner[0]), int(corner[1])), int(5*self.eye_scale_factor), (0, 255, 255), thickness=-1) #3. keep in image bounds, do this even when not dragging because the image video_sizes could change. self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0)) #4. flipping images, converting to greyscale #eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling eyeimage = cv2.resize(self.eye_frames[eye_index].img,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor) if self.mirror[str(eye_index)]: eyeimage = np.fliplr(eyeimage) if self.flip[str(eye_index)]: eyeimage = np.flipud(eyeimage) #eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR) # 5. finally overlay the image x, y = int(self.pos[eye_index][0]), int(self.pos[eye_index][1]) transparent_image_overlay((x, y), eyeimage, frame.img, self.alpha)
def on_window_mouse_button(self, window, button, action, mods): if action == glfw.GLFW_PRESS: self.input["down"] = True self.input["mouse"] = glfw.glfwGetCursorPos(window) if action == glfw.GLFW_RELEASE: self.input["down"] = False
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, version): """Reads world video and runs plugins. Creates a window, gl context. Grabs images from a capture. Maps pupil to gaze data Can run various plug-ins. Reacts to notifications: ``set_detection_mapping_mode`` ``eye_process.started`` ``start_plugin`` Emits notifications: ``eye_process.should_start`` ``eye_process.should_stop`` ``set_detection_mapping_mode`` ``world_process.started`` ``world_process.stopped`` ``recording.should_stop``: Emits on camera failure ``launcher_process.should_stop`` Emits data: ``gaze``: Gaze data from current gaze mapping plugin.`` ``*``: any other plugin generated data in the events that it not [dt,pupil,gaze]. """ # We defer the imports because of multiprocessing. # Otherwise the world process each process also loads the other imports. # This is not harmful but unnecessary. # general imports from time import sleep import logging # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] # logger.setLevel(logging.DEBUG) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) def launch_eye_process(eye_id, delay=0): n = { 'subject': 'eye_process.should_start.{}'.format(eye_id), 'eye_id': eye_id, 'delay': delay } ipc_pub.notify(n) def stop_eye_process(eye_id): n = { 'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id, 'delay': 0.2 } ipc_pub.notify(n) def start_stop_eye(eye_id, make_alive): if make_alive: launch_eye_process(eye_id) else: stop_eye_process(eye_id) def set_detection_mapping_mode(new_mode): n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode} ipc_pub.notify(n) try: # display import glfw from version_utils import VersionFormat from pyglui import ui, cygl, __version__ as pyglui_version assert VersionFormat(pyglui_version) >= VersionFormat( '1.9'), 'pyglui out of date, please upgrade to newest version' from pyglui.cygl.utils import Named_Texture import gl_utils # helpers/utils from file_methods import Persistent_Dict from methods import normalize, denormalize, delta_t, get_system_info, timer from uvc import get_time_monotonic logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) import audio # trigger pupil detector cpp build: import pupil_detectors del pupil_detectors # Plug-ins from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze from time_sync import Time_Sync from pupil_remote import Pupil_Remote from pupil_groups import Pupil_Groups from surface_tracker import Surface_Tracker from log_display import Log_Display from annotations import Annotation_Capture from log_history import Log_History from frame_publisher import Frame_Publisher from blink_detection import Blink_Detection from video_capture import source_classes, manager_classes, Base_Manager, Base_Source from pupil_data_relay import Pupil_Data_Relay from remote_recorder import Remote_Recorder from audio_capture import Audio_Capture from accuracy_visualizer import Accuracy_Visualizer # from saccade_detector import Saccade_Detector from system_graphs import System_Graphs from camera_intrinsics_estimation import Camera_Intrinsics_Estimation from hololens_relay import Hololens_Relay # UI Platform tweaks if platform.system() == 'Linux': scroll_factor = 10.0 window_position_default = (30, 30) elif platform.system() == 'Windows': scroll_factor = 10.0 window_position_default = (8, 31) else: scroll_factor = 1.0 window_position_default = (0, 0) icon_bar_width = 50 window_size = None camera_render_size = None hdpi_factor = 1.0 # g_pool holds variables for this process they are accesible to all plugins g_pool = Global_Container() g_pool.app = 'capture' g_pool.process = 'world' g_pool.user_dir = user_dir g_pool.version = version g_pool.timebase = timebase g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.eyes_are_alive = eyes_are_alive def get_timestamp(): return get_time_monotonic() - g_pool.timebase.value g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic # manage plugins runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, 'plugins')) user_plugins = [ Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker, Annotation_Capture, Log_History, Fixation_Detector, Blink_Detection, Remote_Recorder, Accuracy_Visualizer, Camera_Intrinsics_Estimation, Hololens_Relay ] system_plugins = [ Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs ] + manager_classes + source_classes plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins user_plugins += [ p for p in runtime_plugins if not isinstance(p, (Base_Manager, Base_Source, System_Plugin_Base, Calibration_Plugin, Gaze_Mapping_Plugin)) ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} default_capture_settings = { 'preferred_names': [ "Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e" ], 'frame_size': (1280, 720), 'frame_rate': 30 } default_plugins = [("UVC_Source", default_capture_settings), ('Pupil_Data_Relay', {}), ('UVC_Manager', {}), ('Log_Display', {}), ('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}), ('Screen_Marker_Calibration', {}), ('Recorder', {}), ('Pupil_Remote', {}), ('Plugin_Manager', {}), ('System_Graphs', {})] # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): paths = [paths[x].decode('utf-8') for x in range(count)] for p in g_pool.plugins: p.on_drop(paths) tick = delta_t() def get_dt(): return next(tick) # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version: logger.info( "Session setting are from a different version of this app. I will not use those." ) session_settings.clear() g_pool.detection_mapping_mode = session_settings.get( 'detection_mapping_mode', '3d') g_pool.active_calibration_plugin = None g_pool.active_gaze_mapping_plugin = None g_pool.capture = None audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode) def handle_notifications(n): subject = n['subject'] if subject == 'set_detection_mapping_mode': if n['mode'] == '2d': if ("Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name): logger.warning( "The gaze mapper is not supported in 2d mode. Please recalibrate." ) g_pool.plugins.add( g_pool.plugin_by_name['Dummy_Gaze_Mapper']) g_pool.detection_mapping_mode = n['mode'] elif subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject == 'stop_plugin': for p in g_pool.plugins: if p.class_name == n['name']: p.alive = False g_pool.plugins.clean() elif subject == 'eye_process.started': n = { 'subject': 'set_detection_mapping_mode', 'mode': g_pool.detection_mapping_mode } ipc_pub.notify(n) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': world.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) # window and gl setup glfw.glfwInit() width, height = session_settings.get('window_size', (1280 + icon_bar_width, 720)) main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") window_pos = session_settings.get('window_position', window_position_default) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = camera_render_size[0] + \ int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \ glfw.glfwGetFramebufferSize(main_window)[1] logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'world_process.should_start', 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed # setup GUI g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.6, .8, 1., 1.2, 1.4], label='Interface size')) def set_window_size(): f_width, f_height = g_pool.capture.frame_size f_width += int(icon_bar_width * g_pool.gui.scale) glfw.glfwSetWindowSize(main_window, f_width, f_height) general_settings.append(ui.Button('Reset window size', set_window_size)) general_settings.append( ui.Selector('audio_mode', audio, selection=audio.audio_modes)) general_settings.append( ui.Selector('detection_mapping_mode', g_pool, label='detection & mapping mode', setter=set_detection_mapping_mode, selection=['2d', '3d'])) general_settings.append( ui.Switch('eye0_process', label='Detect eye 0', setter=lambda alive: start_stop_eye(0, alive), getter=lambda: eyes_are_alive[0].value)) general_settings.append( ui.Switch('eye1_process', label='Detect eye 1', setter=lambda alive: start_stop_eye(1, alive), getter=lambda: eyes_are_alive[1].value)) general_settings.append( ui.Info_Text('Capture Version: {}'.format(g_pool.version))) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) # plugins that are loaded based on user settings from previous session g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() toggle_general_settings(False) # now the we have aproper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get('ui_config', {}) # create a timer to control window update frequency window_update_timer = timer(1 / 60) def window_should_update(): return next(window_update_timer) # trigger setup of window and gl sizes on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) if session_settings.get('eye1_process_alive', False): launch_eye_process(1, delay=0.6) if session_settings.get('eye0_process_alive', True): launch_eye_process(0, delay=0.3) ipc_pub.notify({'subject': 'world_process.started'}) logger.warning('Process started.') # Event loop while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) #a dictionary that allows plugins to post and read events events = {} # report time between now and the last loop interation events['dt'] = get_dt() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # send new events to ipc: del events['pupil_positions'] # already on the wire del events['gaze_positions'] # sent earlier if 'frame' in events: del events['frame'] # send explicity with frame publisher if 'depth_frame' in events: del events['depth_frame'] if 'audio_packets' in events: del events['audio_packets'] del events['dt'] # no need to send this for topic, data in events.items(): assert (isinstance(data, (list, tuple))) for d in data: ipc_pub.send(topic, d) glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if window_should_update() and gl_utils.is_window_visible( main_window): gl_utils.glViewport(0, 0, *camera_render_size) for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) unused_elements = g_pool.gui.update() for button, action, mods in unused_elements.buttons: x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in unused_elements.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in unused_elements.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() glfw.glfwRestoreWindow(main_window) # need to do this for windows os session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings['eye0_process_alive'] = eyes_are_alive[0].value session_settings['eye1_process_alive'] = eyes_are_alive[1].value session_settings[ 'detection_mapping_mode'] = g_pool.detection_mapping_mode session_settings['audio_mode'] = audio.audio_mode session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) glfw.glfwTerminate() except: import traceback trace = traceback.format_exc() logger.error('Process Capture crashed with trace:\n{}'.format(trace)) finally: # shut down eye processes: stop_eye_process(0) stop_eye_process(1) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'world_process.stopped'}) sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports import logging import errno from glob import glob from copy import deepcopy from time import time # networking import zmq import zmq_tools # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',)) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) # imports from file_methods import Persistent_Dict, load_object import numpy as np # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, graph, cygl from pyglui.cygl.utils import Named_Texture import gl_utils # capture from video_capture import File_Source, EndofVideoFileError, FileSeekError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # monitoring import psutil # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_bar import Seek_Bar from trim_marks import Trim_Marks from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector # from manual_gaze_correction import Manual_Gaze_Correction from batch_exporter import Batch_Exporter from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration assert pyglui_version >= '1.5' runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins')) system_plugins = [Log_Display, Seek_Bar, Trim_Marks] user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration] + runtime_plugins available_plugins = system_plugins + user_launchable_plugins name_by_index = [p.__name__ for p in available_plugins] plugin_by_name = dict(zip(name_by_index, available_plugins)) # Callback functions def on_resize(window, w, h): if gl_utils.is_window_visible(window): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: g.scale = hdpi_factor g.adjust_window_size(w, h) gl_utils.adjust_gl_view(w, h) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_char(window, char): g_pool.gui.update_char(char) def on_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0]) g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y*scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug("Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir}) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format(new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) timestamps = np.load(timestamps_path) # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url # Initialize capture cap = File_Source(g_pool, video_path, timestamps=list(timestamps)) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info("Session setting are a different version of this app. I will not use those.") session_settings.clear() width, height = session_settings.get('window_size', cap.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() def set_scale(new_scale): g_pool.gui_user_scale = new_scale on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.capture = cap g_pool.timestamps = timestamps g_pool.get_timestamp = lambda: 0. g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index() + 1) except(FileSeekError): logger.warning("Could not seek to next frame.") else: g_pool.new_seek = True def prev_frame(_): try: cap.seek_to_frame(cap.get_frame_index() - 1) except(FileSeekError): logger.warning("Could not seek to previous frame.") else: g_pool.new_seek = True def toggle_play(new_state): if cap.get_frame_index() >= cap.get_frame_count()-5: cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause. logger.warning("End of video - restart at beginning.") g_pool.play = new_state def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time()+.8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_launchable_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir} g_pool.ipc_pub.notify(notification) g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560)) g_pool.main_menu.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))) g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size')) g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version))) g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version']))) g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version']))) g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) g_pool.main_menu.append(ui.Info_Text('Open plugins')) selector_label = "Select to load" def append_selector(label, plugins): plugins.sort(key=lambda p: p.__name__) plugin_labels = [p.__name__.replace('_', ' ') for p in plugins] g_pool.main_menu.append(ui.Selector(label, selection=[selector_label] + plugins, labels=[selector_label] + plugin_labels, setter=open_plugin, getter=lambda: selector_label)) base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base] base_labels = ['Visualizer:', 'Analyser:', 'Data Source:'] launchable = user_launchable_plugins.copy() for base_class, label in zip(base_plugins, base_labels): member_plugins = [] for p in user_launchable_plugins: if issubclass(p, base_class): member_plugins.append(p) launchable.remove(p) append_selector(label, member_plugins) # launchable only contains plugins that could not be assigned to any of the above categories append_selector('Other', launchable) g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins)) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100)) g_pool.play_button = ui.Thumb('play', g_pool, label=chr(0xf04b), setter=toggle_play, hotkey=glfw.GLFW_KEY_SPACE, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.play_button.on_color[:] = (0, 1., .0, .8) g_pool.forward_button = ui.Thumb('forward', label=chr(0xf04e), getter=lambda: False, setter=next_frame, hotkey=glfw.GLFW_KEY_RIGHT, label_font='fontawesome', label_offset_x=5, label_offset_y=0, label_offset_size=-24) g_pool.backward_button = ui.Thumb('backward', label=chr(0xf04a), getter=lambda: False, setter=prev_frame, hotkey=glfw.GLFW_KEY_LEFT, label_font='fontawesome', label_offset_x=-5, label_offset_y=0, label_offset_size=-24) g_pool.export_button = ui.Thumb('export', label=chr(0xf063), getter=lambda: False, setter=do_export, hotkey='e', label_font='fontawesome', label_offset_x=0, label_offset_y=2, label_offset_size=-24) g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button]) g_pool.gui.append(g_pool.quickbar) g_pool.gui.append(g_pool.main_menu) # we always load these plugins system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})] default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] previous_plugins = session_settings.get('loaded_plugins', default_plugins) g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_key) glfw.glfwSetCharCallback(main_window, on_char) glfw.glfwSetMouseButtonCallback(main_window, on_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # set up performace graphs: pid = os.getpid() ps = psutil.Process(pid) ts = None cpu_graph = graph.Bar_Graph() cpu_graph.pos = (20, 110) cpu_graph.update_fn = ps.cpu_percent cpu_graph.update_rate = 5 cpu_graph.label = 'CPU %0.1f' fps_graph = graph.Bar_Graph() fps_graph.pos = (140, 110) fps_graph.update_rate = 5 fps_graph.label = "%0.0f REC FPS" pupil_graph = graph.Bar_Graph(max_val=1.0) pupil_graph.pos = (260, 110) pupil_graph.update_rate = 5 pupil_graph.label = "Confidence: %0.2f" g_pool.graphs = [cpu_graph, fps_graph, pupil_graph] # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add( plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__}) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__}) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = cap.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.play = False logger.warning("end of video") update_graph = True else: update_graph = False frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # new positons we make a deepcopy just like the image is a copy. events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index]) events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index]) if update_graph: # update performace graphs for p in events['pupil_positions']: pupil_graph.add(p['confidence']) t = new_frame.timestamp if ts and ts != t: dt, ts = t-ts, t fps_graph.add(1./dt) else: ts = new_frame.timestamp g_pool.play_button.status_text = str(frame.index) # always update the CPU graph cpu_graph.update() # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() # render camera image glfw.glfwMakeContextCurrent(main_window) gl_utils.make_coord_system_norm_based() g_pool.image_tex.update_from_frame(frame) g_pool.image_tex.draw() gl_utils.make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() fps_graph.draw() cpu_graph.draw() pupil_graph.draw() unused_buttons = g_pool.gui.update() for b in unused_buttons: button,action,mods = b pos = glfw.glfwGetCursorPos(main_window) pos = normalize(pos, glfw.glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) # present frames at appropriate speed cap.wait(frame) glfw.glfwSwapBuffers(main_window) glfw.glfwPollEvents() session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos(main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() cap.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'})
def recent_events(self, events): frame = events.get('frame') if not frame: return for eye_index in self.showeyes: requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index] #1. do we need a new frame? if requested_eye_frame_idx != self.eye_frames[eye_index].index: # do we need to seek? if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1: # if we just need to seek by one frame, its faster to just read one and and throw it away. _ = self.eye_cap[eye_index].get_frame() if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index(): # only now do I need to seek self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx) # reading the new eye frame frame try: self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame() except EndofVideoFileError: logger.warning("Reached the end of the eye video for eye video {}.".format(eye_index)) else: #our old frame is still valid because we are doing upsampling pass #2. dragging image if self.drag_offset[eye_index] is not None: pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0] self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1] else: self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)] #3. keep in image bounds, do this even when not dragging because the image video_sizes could change. self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0)) #4. flipping images, converting to greyscale eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling eyeimage = cv2.resize(eye_gray,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor) if self.mirror[str(eye_index)]: eyeimage = np.fliplr(eyeimage) if self.flip[str(eye_index)]: eyeimage = np.flipud(eyeimage) eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR) if self.show_ellipses and events['pupil_positions']: for pd in events['pupil_positions']: if pd['id'] == eye_index and pd['timestamp'] == self.eye_frames[eye_index].timestamp: el = pd['ellipse'] conf = int(pd.get('model_confidence', pd.get('confidence', 0.1)) * 255) center = list(map(lambda val: int(self.eye_scale_factor*val), el['center'])) el['axes'] = tuple(map(lambda val: int(self.eye_scale_factor*val/2), el['axes'])) el['angle'] = int(el['angle']) el_points = cv2.ellipse2Poly(tuple(center), el['axes'], el['angle'], 0, 360, 1) if self.mirror[str(eye_index)]: el_points = [(self.video_size[0] - x, y) for x, y in el_points] center[0] = self.video_size[0] - center[0] if self.flip[str(eye_index)]: el_points = [(x, self.video_size[1] - y) for x, y in el_points] center[1] = self.video_size[1] - center[1] cv2.polylines(eyeimage, [np.asarray(el_points)], True, (0, 0, 255, conf), thickness=math.ceil(2*self.eye_scale_factor)) cv2.circle(eyeimage, tuple(center), int(5*self.eye_scale_factor), (0, 0, 255, conf), thickness=-1) # 5. finally overlay the image x, y = int(self.pos[eye_index][0]), int(self.pos[eye_index][1]) transparent_image_overlay((x, y), eyeimage, frame.img, self.alpha)