def recent_events(self, events): frame = events.get('frame') if not frame: return self.img_shape = frame.img.shape self.update_marker_cache() # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter] self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher( frame.index ) # tell precacher that it better have every thing from here on analyzed events['surfaces'] = [] # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers, self.camera_calibration, self.min_marker_perimeter, self.min_id_confidence) if s.detected: events['surfaces'].append({ 'name': s.name, 'uid': s.uid, 'm_to_screen': s.m_to_screen, 'm_from_screen': s.m_from_screen, 'timestamp': frame.timestamp }) if self.mode == "Show marker IDs": draw_markers(frame.img, self.markers) elif self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window), flip_y=True) for s, v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache, self.camera_calibration, self.min_marker_perimeter, self.min_id_confidence) self.notify_all({ 'subject': 'surfaces_changed', 'delay': 1 }) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def update(self,frame,events): self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray if self.robust_detection: self.markers = detect_markers_robust(gray, grid_size = 5, prev_markers=self.markers, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0, true_detect_every_frame=3, invert_image=self.invert_image) else: self.markers = detect_markers(gray, grid_size = 5, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0, invert_image=self.invert_image) if self.mode == "Show marker IDs": draw_markers(frame.gray,self.markers) events['surface'] = [] # locate surfaces for s in self.surfaces: s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d) if s.detected: events['surface'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(), 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist()}) if self.running: self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze_positions',[]): gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s)
def update(self, frame, recent_pupil_positions, events): self.img = frame.img self.img_shape = frame.img.shape self.update_marker_cache() self.markers = self.cache[frame.index] if self.markers == False: # locate markers because precacher has not anayzed this frame yet. Most likely a seek event self.markers = [] self.seek_marker_cacher( frame.index ) # tell precacher that it better have every thing from here analyzed # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers) if s.detected: events.append({ 'type': 'marker_ref_surface', 'name': s.name, 'uid': s.uid, 'm_to_screen': s.m_to_screen, 'm_from_screen': s.m_from_screen, 'timestamp': frame.timestamp, 'gaze_on_srf': s.gaze_on_srf }) if self.draw_markers.value: draw_markers(frame.img, self.markers) # edit surfaces by user if self.surface_edit_mode: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for s, v_idx in self.edit_surfaces: if s.detected: pos = normalize(pos, (self.img_shape[1], self.img_shape[0]), flip_y=True) new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) s.cache = None else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape if self.robust_detection.value: self.markers = detect_markers_robust(img, grid_size = 5, prev_markers=self.markers, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture.value, visualize=0, true_detect_every_frame=3) else: self.markers = detect_markers_simple(img, grid_size = 5, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture.value, visualize=0) # locate surfaces for s in self.surfaces: s.locate(self.markers) if s.detected: events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.draw_markers.value: draw_markers(img,self.markers) # edit surfaces by user if self.surface_edit_mode: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for s,v_idx in self.edit_surfaces: if s.detected: pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True) new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in recent_pupil_positions: if p['norm_pupil'] is not None: gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) p['realtime gaze on '+s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def update(self, frame, events): self.img_shape = frame.height, frame.width, 3 if self.running: gray = frame.gray if self.robust_detection: self.markers = detect_markers_robust( gray, grid_size=5, prev_markers=self.markers, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0, true_detect_every_frame=3, ) else: self.markers = detect_markers( gray, grid_size=5, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture, visualize=0, ) if self.mode == "Show marker IDs": draw_markers(frame.img, self.markers) # locate surfaces for s in self.surfaces: s.locate(self.markers, self.locate_3d, self.camera_intrinsics) # if s.detected: # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.running: self.button.status_text = "%s/%s" % (len([s for s in self.surfaces if s.detected]), len(self.surfaces)) else: self.button.status_text = "tracking paused" # edit surfaces by user if self.mode == "Surface edit mode": window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window), flip_y=True) for s, v_idx in self.edit_surfaces: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) # map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get("gaze_positions", []): gp_on_s = tuple(s.img_to_ref_surface(np.array(p["norm_pos"]))) p["realtime gaze on " + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s)
def update(self,frame,events): self.img_shape = frame.img.shape self.update_marker_cache() # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter] self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed events['surfaces'] = [] # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter) if s.detected: events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) elif self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None and s not in [s for s,i in self.edit_surf_verts]: s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter) self.notify_all_delayed({'subject':'surfaces_changed'}) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze_positions',[]): gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def recent_events(self, events): frame = events.get('frame') if not frame: return self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray if self.invert_image: gray = 255-gray if self.robust_detection: self.markers = detect_markers_robust( gray, grid_size = 5,aperture=self.aperture, prev_markers=self.markers, true_detect_every_frame=3, min_marker_perimeter=self.min_marker_perimeter) else: self.markers = detect_markers( gray, grid_size = 5,aperture=self.aperture, min_marker_perimeter=self.min_marker_perimeter) if self.mode == "Show marker IDs": draw_markers(frame.gray,self.markers) # locate surfaces, map gaze for s in self.surfaces: s.locate(self.markers,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d) if s.detected: s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen) s.update_gaze_history() else: s.gaze_on_srf =[] events['surfaces'] = [] for s in self.surfaces: if s.detected: events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(),'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None}) if self.running: self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos)
def update(self,frame,events): self.img = frame.img self.img_shape = frame.img.shape self.update_marker_cache() self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers) if s.detected: pass # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf}) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) # edit surfaces by user if self.mode == "Surface edit mode": window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surfaces: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) s.cache = None self.heatmap = None else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze_positions',[]): gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def recent_events(self, events): frame = events.get('frame') if not frame: return self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray if self.invert_image: gray = 255-gray if self.robust_detection: self.markers = detect_markers_robust( gray, grid_size = 5,aperture=self.aperture, prev_markers=self.markers, true_detect_every_frame=3, min_marker_perimeter=self.min_marker_perimeter) else: self.markers = detect_markers( gray, grid_size = 5,aperture=self.aperture, min_marker_perimeter=self.min_marker_perimeter) if self.mode == "Show marker IDs": draw_markers(frame.gray,self.markers) # locate surfaces, map gaze for s in self.surfaces: s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d) if s.detected: s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen) else: s.gaze_on_srf =[] events['surfaces'] = [] for s in self.surfaces: if s.detected: events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(),'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None}) if self.running: self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos)
def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape if self.robust_detection.value: self.markers = detect_markers_robust(img,grid_size = 5, prev_markers=self.markers, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture.value, visualize=0, true_detect_every_frame=3) else: self.markers = detect_markers_simple(img,grid_size = 5,min_marker_perimeter=self.min_marker_perimeter,aperture=self.aperture.value,visualize=0) if self.draw_markers.value: draw_markers(img,self.markers) # print self.markers for s in self.surfaces: s.locate(self.markers) if s.detected: events.append({'type':'marker_ref_surface','name':s.name,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.surface_edit_mode: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for s,v_idx in self.edit_surfaces: if s.detected: pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True) new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for p in recent_pupil_positions: if p['norm_pupil'] is not None: for s in self.surfaces: if s.detected: p['realtime gaze on '+s.name] = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) if self._window: # save a local copy for when we display gaze for debugging on ref surface self.recent_pupil_positions = recent_pupil_positions if self.window_should_close: self.close_window() if self.window_should_open: self.open_window()
def recent_events(self,events): if 'frame' in events: frame = events['frame'] self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray # hack "self.markers" instead "self.screens" is keeped for inheritence compatibility self.markers = detect_screens(gray) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) events['frame'] = frame # locate surfaces, map gaze for s in self.surfaces: s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d) if s.detected: s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen) else: s.gaze_on_srf =[] events['surface'] = [] for s in self.surfaces: if s.detected: events['surface'].append({ 'name':s.name, 'uid':s.uid, 'm_to_screen':s.m_to_screen.tolist(), 'm_from_screen':s.m_from_screen.tolist(), 'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp, 'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None }) if self.running: self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos)
def recent_events(self,events): frame = events.get('frame') if not frame: return self.img_shape = frame.img.shape self.update_marker_cache() # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter] self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed events['surfaces'] = [] # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence) if s.detected: events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(),'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None}) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) elif self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence) self.notify_all({'subject':'surfaces_changed','delay':1}) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def update(self, frame, events): self.img = frame.img self.img_shape = frame.img.shape self.update_marker_cache() self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher( frame.index ) # tell precacher that it better have every thing from here on analyzed # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers) if s.detected: pass # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf}) if self.mode == "Show marker IDs": draw_markers(frame.img, self.markers) # edit surfaces by user if self.mode == "Surface edit mode": window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window), flip_y=True) for s, v_idx in self.edit_surfaces: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) s.cache = None self.heatmap = None else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def update(self,frame,recent_pupil_positions,events): self.img = frame.img self.img_shape = frame.img.shape self.update_marker_cache() self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers) if s.detected: events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf}) if self.mode.value == 4: draw_markers(frame.img,self.markers) # edit surfaces by user if self.mode.value == 1: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for s,v_idx in self.edit_surfaces: if s.detected: pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True) new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) s.cache = None self.heatmap = None else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def update(self,frame,events): self.img_shape = frame.height,frame.width,3 if self.running: gray = frame.gray # hack "self.markers" instead "self.screens" is keeped for inheritence compatibility self.markers = detect_screens(gray) if self.mode == "Show marker IDs": draw_markers(frame.img,self.markers) events['surfaces'] = [] # locate surfaces for s in self.surfaces: s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d) if s.detected: events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.running: self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces)) else: self.button.status_text = 'tracking paused' if self.mode == 'Show Markers and Surfaces': # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window),flip_y=True) for s,v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze_positions',[]): gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s)
def gl_display(self): if self.running: #apply undistord image to draw glMatrixMode( GL_MODELVIEW ) make_coord_system_norm_based() apply_gl_texture(self.frame_img) if self.show_undistord: make_coord_system_pixel_based([self.roi[3], self.roi[2], 3]) else: make_coord_system_pixel_based(self.img_shape) if self.mode == "Show marker IDs": visible = [m for m in self.markers if m['visible']] if self.show_undistord: draw_markers(self.img_shape, visible, self.roi) else: draw_markers(self.img_shape, visible) if self.mode == "Draw obj" and self.frame_img != None: if not self.glIsInit : glInit() self.glIsInit = True for m in self.markers: if m['visible']: if m['to_draw']: #drawIn2D( self.camera_intrinsics[0], self.camera_intrinsics[1], m ) if self.show_undistord: glDrawFromCamera( self.camera_intrinsics[0], self.camera_intrinsics[1], m['rot'], m['trans'], self.img_shape, self.roi, m['obj'] ) else: glDrawFromCamera( self.camera_intrinsics[0], self.camera_intrinsics[1], m['rot'], m['trans'], self.img_shape, None, m['obj'] ) self.frame_img = None
def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape if self.robust_detection.value: self.markers = detect_markers_robust(img, grid_size = 5, prev_markers=self.markers, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture.value, visualize=0, true_detect_every_frame=3) else: self.markers = detect_markers_simple(img, grid_size = 5, min_marker_perimeter=self.min_marker_perimeter, aperture=self.aperture.value, visualize=0) # locate surfaces for s in self.surfaces: s.locate(self.markers) if s.detected: events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) if self.draw_markers.value: draw_markers(img,self.markers) # edit surfaces by user if self.surface_edit_mode: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for s,v_idx in self.edit_surfaces: if s.detected: pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True) new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in recent_pupil_positions: if p['norm_pupil'] is not None: gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) p['realtime gaze on '+s.name] = gp_on_s print gp_on_s #try: # ser = serial.Serial("/dev/ttyACM0") # ser.close() # ser.open() # print(ser.read()) # s.gaze_on_srf.append(gp_on_s) #except: # print("it didn't work!!!!!") #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def recent_events(self, events): frame = events.get("frame") if not frame: return self.img_shape = frame.height, frame.width, 3 if self.running: gray = frame.gray if self.invert_image: gray = 255 - gray if self.robust_detection: self.markers = detect_markers_robust( gray, grid_size=5, aperture=self.aperture, prev_markers=self.markers, true_detect_every_frame=3, min_marker_perimeter=self.min_marker_perimeter, ) else: self.markers = detect_markers( gray, grid_size=5, aperture=self.aperture, min_marker_perimeter=self.min_marker_perimeter, ) if self.mode == "Show marker IDs": draw_markers(frame.gray, self.markers) # locate surfaces, map gaze for s in self.surfaces: s.locate( self.markers, self.min_marker_perimeter, self.min_id_confidence, self.locate_3d, ) if s.detected: s.gaze_on_srf = s.map_data_to_surface(events.get("gaze", []), s.m_from_screen) s.fixations_on_srf = s.map_data_to_surface( events.get("fixations", []), s.m_from_screen) s.update_gaze_history() else: s.gaze_on_srf = [] s.fixations_on_srf = [] events["surfaces"] = [] for s in self.surfaces: if s.detected: datum = { "topic": "surfaces.{}".format(s.name), "name": s.name, "uid": s.uid, "m_to_screen": s.m_to_screen.tolist(), "m_from_screen": s.m_from_screen.tolist(), "gaze_on_srf": s.gaze_on_srf, "fixations_on_srf": s.fixations_on_srf, "timestamp": frame.timestamp, "camera_pose_3d": s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None, } events["surfaces"].append(datum) if self.running: self.button.status_text = "{}/{}".format( len([s for s in self.surfaces if s.detected]), len(self.surfaces)) else: self.button.status_text = "tracking paused" if self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: pos = self._last_mouse_pos for s, v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos)
def recent_events(self, events): frame = events.get('frame') if not frame: return self.img_shape = frame.img.shape self.update_marker_cache() # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter] self.markers = self.cache[frame.index] if self.markers is False: self.markers = [] # tell precacher that it better have every thing from here on analyzed self.seek_marker_cacher(frame.index) events['surfaces'] = [] # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers, self.min_marker_perimeter, self.min_id_confidence) if s.detected: events['surfaces'].append({ 'name': s.name, 'uid': s.uid, 'm_to_screen': s.m_to_screen.tolist(), 'm_from_screen': s.m_from_screen.tolist(), 'gaze_on_srf': s.gaze_on_srf, 'timestamp': frame.timestamp, 'camera_pose_3d': s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None }) if self.mode == "Show marker IDs": draw_markers(frame.img, self.markers) elif self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: pos = self._last_mouse_pos # inherited from Surface_Tracker for s, v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache, self.min_marker_perimeter, self.min_id_confidence) self.notify_all({ 'subject': 'surfaces_changed', 'delay': 1 }) # allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def marker_positions(camera_spec, videofile, outfile, new_camera=None, start_time=0.0, end_time=float("inf"), visualize=False, output_camera=None, write=False): camera = pickle.load(open(camera_spec, 'rb'), encoding='bytes') image_resolution = camera[b'resolution'] if b'rect_map' not in camera: camera_matrix = camera[b'camera_matrix'] camera_distortion = camera[b'dist_coefs'] rect_camera_matrix, roi = cv2.getOptimalNewCameraMatrix( camera_matrix, camera_distortion, image_resolution, 0.0) rmap = cv2.initUndistortRectifyMap(camera_matrix, camera_distortion, None, rect_camera_matrix, image_resolution, cv2.CV_32FC1) else: rmap = camera[b'rect_map'] rect_camera_matrix = camera[b'rect_camera_matrix'] K, D, resolution, cm = camera[b'camera_matrix'], camera[ b'dist_coefs'], camera[b'resolution'], rect_camera_matrix camera = {} camera['camera_matrix'] = rect_camera_matrix camera['dist_coefs'] = None camera['resolution'] = image_resolution if new_camera is not None: save_object(camera, new_camera) #rectify_gaze_data(path, K, D, rect_camera_matrix) #if new_camera is not None: # pickle.dump(camera, open(new_camera, 'w')) video = cv2.VideoCapture(videofile) video.set(cv2.CAP_PROP_POS_MSEC, start_time * 1000) frames = [] #marker_tracker = markerdetect.MarkerTracker() prev_minute = 0.0 marker_cache = [] #fourcc = cv2.VideoWriter_fourcc(*'XVID') #if write == True: # out = cv2.VideoWriter(path + "world.mp4",fourcc, 30.0, (1280,720)) while True: ret, oframe = video.read() if not ret: break frame = cv2.remap(oframe, rmap[0], rmap[1], cv2.INTER_LINEAR) #if write == True: # out.write(frame) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) msecs = video.get(cv2.CAP_PROP_POS_MSEC) time = msecs / 1000.0 if time > end_time: break markers = markerdetect.detect_markers(frame, 5, min_marker_perimeter=15) marker_cache.append(markers) frame_d = { 'ts': time, 'markers': markers, } frames.append(frame_d) if not visualize: continue markerdetect.draw_markers(frame, frame_d['markers']) cv2.imshow('frameBG', frame) cv2.waitKey(1) np.save(outfile, frames) if write == True: out.release() return marker_cache
def update(self, frame, events): self.img_shape = frame.img.shape self.update_marker_cache() # self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter] self.markers = self.cache[frame.index] if self.markers == False: self.markers = [] self.seek_marker_cacher( frame.index ) # tell precacher that it better have every thing from here on analyzed events['surfaces'] = [] # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): s.locate(self.markers, self.camera_calibration, self.min_marker_perimeter) if s.detected: events['surfaces'].append({ 'name': s.name, 'uid': s.uid, 'm_to_screen': s.m_to_screen, 'm_from_screen': s.m_from_screen, 'timestamp': frame.timestamp }) if self.mode == "Show marker IDs": draw_markers(frame.img, self.markers) elif self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window), flip_y=True) for s, v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos) else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None and s not in [ s for s, i in self.edit_surf_verts ]: s.init_cache(self.cache, self.camera_calibration, self.min_marker_perimeter) self.notify_all({ 'subject': 'surfaces_changed', 'delay': 1 }) #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: s.gaze_on_srf = [] for p in events.get('gaze_positions', []): gp_on_s = tuple( s.img_to_ref_surface(np.array(p['norm_pos']))) p['realtime gaze on ' + s.name] = gp_on_s s.gaze_on_srf.append(gp_on_s) #allow surfaces to open/close windows for s in self.surfaces: if s.window_should_close: s.close_window() if s.window_should_open: s.open_window()
def update(self, frame, events): self.img_shape = frame.height, frame.width, 3 if self.running: gray = frame.gray if self.invert_image: gray = 255 - gray if self.robust_detection: self.markers = detect_markers_robust( gray, grid_size=5, aperture=self.aperture, prev_markers=self.markers, true_detect_every_frame=3, min_marker_perimeter=self.min_marker_perimeter, ) else: self.markers = detect_markers( gray, grid_size=5, aperture=self.aperture, min_marker_perimeter=self.min_marker_perimeter ) if self.mode == "Show marker IDs": draw_markers(frame.gray, self.markers) # locate surfaces, map gaze for s in self.surfaces: s.locate( self.markers, self.camera_calibration, self.min_marker_perimeter, self.min_id_confidence, self.locate_3d ) if s.detected: s.gaze_on_srf = s.map_data_to_surface(events.get("gaze_positions", []), s.m_from_screen) else: s.gaze_on_srf = [] events["surface"] = [] for s in self.surfaces: if s.detected: events["surface"].append( { "name": s.name, "uid": s.uid, "m_to_screen": s.m_to_screen.tolist(), "m_from_screen": s.m_from_screen.tolist(), "gaze_on_srf": s.gaze_on_srf, "timestamp": frame.timestamp, "camera_pose_3d": s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None, } ) if self.running: self.button.status_text = "%s/%s" % (len([s for s in self.surfaces if s.detected]), len(self.surfaces)) else: self.button.status_text = "tracking paused" if self.mode == "Show Markers and Surfaces": # edit surfaces by user if self.edit_surf_verts: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window), flip_y=True) for s, v_idx in self.edit_surf_verts: if s.detected: new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx, new_pos)