def recent_events(self, events):
frame = events.get('frame')
if not frame:
return
self.img_shape = frame.img.shape
self.update_marker_cache()
# self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(
frame.index
) # tell precacher that it better have every thing from here on analyzed
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers, self.camera_calibration,
self.min_marker_perimeter, self.min_id_confidence)
if s.detected:
events['surfaces'].append({
'name': s.name,
'uid': s.uid,
'm_to_screen': s.m_to_screen,
'm_from_screen': s.m_from_screen,
'timestamp': frame.timestamp
})
if self.mode == "Show marker IDs":
draw_markers(frame.img, self.markers)
elif self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
for s, v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache, self.camera_calibration,
self.min_marker_perimeter,
self.min_id_confidence)
self.notify_all({
'subject': 'surfaces_changed',
'delay': 1
})
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def update(self,frame,events):
self.img_shape = frame.height,frame.width,3
if self.running:
gray = frame.gray
if self.robust_detection:
self.markers = detect_markers_robust(gray,
grid_size = 5,
prev_markers=self.markers,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture,
visualize=0,
true_detect_every_frame=3,
invert_image=self.invert_image)
else:
self.markers = detect_markers(gray,
grid_size = 5,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture,
visualize=0,
invert_image=self.invert_image)
if self.mode == "Show marker IDs":
draw_markers(frame.gray,self.markers)
events['surface'] = []
# locate surfaces
for s in self.surfaces:
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d)
if s.detected:
events['surface'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(), 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist()})
if self.running:
self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces))
else:
self.button.status_text = 'tracking paused'
if self.mode == 'Show Markers and Surfaces':
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get('gaze_positions',[]):
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
p['realtime gaze on ' + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
def update(self, frame, recent_pupil_positions, events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
# locate markers because precacher has not anayzed this frame yet. Most likely a seek event
self.markers = []
self.seek_marker_cacher(
frame.index
) # tell precacher that it better have every thing from here analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
events.append({
'type': 'marker_ref_surface',
'name': s.name,
'uid': s.uid,
'm_to_screen': s.m_to_screen,
'm_from_screen': s.m_from_screen,
'timestamp': frame.timestamp,
'gaze_on_srf': s.gaze_on_srf
})
if self.draw_markers.value:
draw_markers(frame.img, self.markers)
# edit surfaces by user
if self.surface_edit_mode:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # Position in img pixels
for s, v_idx in self.edit_surfaces:
if s.detected:
pos = normalize(pos,
(self.img_shape[1], self.img_shape[0]),
flip_y=True)
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
s.cache = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def update(self,frame,recent_pupil_positions,events):
img = frame.img
self.img_shape = frame.img.shape
if self.robust_detection.value:
self.markers = detect_markers_robust(img,
grid_size = 5,
prev_markers=self.markers,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture.value,
visualize=0,
true_detect_every_frame=3)
else:
self.markers = detect_markers_simple(img,
grid_size = 5,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture.value,
visualize=0)
# locate surfaces
for s in self.surfaces:
s.locate(self.markers)
if s.detected:
events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.draw_markers.value:
draw_markers(img,self.markers)
# edit surfaces by user
if self.surface_edit_mode:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for s,v_idx in self.edit_surfaces:
if s.detected:
pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True)
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in recent_pupil_positions:
if p['norm_pupil'] is not None:
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze'])))
p['realtime gaze on '+s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def update(self, frame, events):
self.img_shape = frame.height, frame.width, 3
if self.running:
gray = frame.gray
if self.robust_detection:
self.markers = detect_markers_robust(
gray,
grid_size=5,
prev_markers=self.markers,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture,
visualize=0,
true_detect_every_frame=3,
)
else:
self.markers = detect_markers(
gray,
grid_size=5,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture,
visualize=0,
)
if self.mode == "Show marker IDs":
draw_markers(frame.img, self.markers)
# locate surfaces
for s in self.surfaces:
s.locate(self.markers, self.locate_3d, self.camera_intrinsics)
# if s.detected:
# events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.running:
self.button.status_text = "%s/%s" % (len([s for s in self.surfaces if s.detected]), len(self.surfaces))
else:
self.button.status_text = "tracking paused"
# edit surfaces by user
if self.mode == "Surface edit mode":
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
for s, v_idx in self.edit_surfaces:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
# map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get("gaze_positions", []):
gp_on_s = tuple(s.img_to_ref_surface(np.array(p["norm_pos"])))
p["realtime gaze on " + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
def update(self,frame,events):
self.img_shape = frame.img.shape
self.update_marker_cache()
# self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter)
if s.detected:
events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
elif self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None and s not in [s for s,i in self.edit_surf_verts]:
s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter)
self.notify_all_delayed({'subject':'surfaces_changed'})
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get('gaze_positions',[]):
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
p['realtime gaze on ' + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def recent_events(self, events):
frame = events.get('frame')
if not frame:
return
self.img_shape = frame.height,frame.width,3
if self.running:
gray = frame.gray
if self.invert_image:
gray = 255-gray
if self.robust_detection:
self.markers = detect_markers_robust(
gray, grid_size = 5,aperture=self.aperture,
prev_markers=self.markers,
true_detect_every_frame=3,
min_marker_perimeter=self.min_marker_perimeter)
else:
self.markers = detect_markers(
gray, grid_size = 5,aperture=self.aperture,
min_marker_perimeter=self.min_marker_perimeter)
if self.mode == "Show marker IDs":
draw_markers(frame.gray,self.markers)
# locate surfaces, map gaze
for s in self.surfaces:
s.locate(self.markers,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d)
if s.detected:
s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen)
s.update_gaze_history()
else:
s.gaze_on_srf =[]
events['surfaces'] = []
for s in self.surfaces:
if s.detected:
events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(),'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None})
if self.running:
self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces))
else:
self.button.status_text = 'tracking paused'
if self.mode == 'Show Markers and Surfaces':
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
def update(self,frame,events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
pass
# events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
# edit surfaces by user
if self.mode == "Surface edit mode":
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surfaces:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
s.cache = None
self.heatmap = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get('gaze_positions',[]):
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
p['realtime gaze on ' + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def recent_events(self, events):
frame = events.get('frame')
if not frame:
return
self.img_shape = frame.height,frame.width,3
if self.running:
gray = frame.gray
if self.invert_image:
gray = 255-gray
if self.robust_detection:
self.markers = detect_markers_robust(
gray, grid_size = 5,aperture=self.aperture,
prev_markers=self.markers,
true_detect_every_frame=3,
min_marker_perimeter=self.min_marker_perimeter)
else:
self.markers = detect_markers(
gray, grid_size = 5,aperture=self.aperture,
min_marker_perimeter=self.min_marker_perimeter)
if self.mode == "Show marker IDs":
draw_markers(frame.gray,self.markers)
# locate surfaces, map gaze
for s in self.surfaces:
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d)
if s.detected:
s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen)
else:
s.gaze_on_srf =[]
events['surfaces'] = []
for s in self.surfaces:
if s.detected:
events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(),'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None})
if self.running:
self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces))
else:
self.button.status_text = 'tracking paused'
if self.mode == 'Show Markers and Surfaces':
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
def update(self,frame,recent_pupil_positions,events):
img = frame.img
self.img_shape = frame.img.shape
if self.robust_detection.value:
self.markers = detect_markers_robust(img,grid_size = 5,
prev_markers=self.markers,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture.value,
visualize=0,
true_detect_every_frame=3)
else:
self.markers = detect_markers_simple(img,grid_size = 5,min_marker_perimeter=self.min_marker_perimeter,aperture=self.aperture.value,visualize=0)
if self.draw_markers.value:
draw_markers(img,self.markers)
# print self.markers
for s in self.surfaces:
s.locate(self.markers)
if s.detected:
events.append({'type':'marker_ref_surface','name':s.name,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.surface_edit_mode:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for s,v_idx in self.edit_surfaces:
if s.detected:
pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True)
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
#map recent gaze onto detected surfaces used for pupil server
for p in recent_pupil_positions:
if p['norm_pupil'] is not None:
for s in self.surfaces:
if s.detected:
p['realtime gaze on '+s.name] = tuple(s.img_to_ref_surface(np.array(p['norm_gaze'])))
if self._window:
# save a local copy for when we display gaze for debugging on ref surface
self.recent_pupil_positions = recent_pupil_positions
if self.window_should_close:
self.close_window()
if self.window_should_open:
self.open_window()
def recent_events(self,events):
if 'frame' in events:
frame = events['frame']
self.img_shape = frame.height,frame.width,3
if self.running:
gray = frame.gray
# hack "self.markers" instead "self.screens" is keeped for inheritence compatibility
self.markers = detect_screens(gray)
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
events['frame'] = frame
# locate surfaces, map gaze
for s in self.surfaces:
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d)
if s.detected:
s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen)
else:
s.gaze_on_srf =[]
events['surface'] = []
for s in self.surfaces:
if s.detected:
events['surface'].append({
'name':s.name,
'uid':s.uid,
'm_to_screen':s.m_to_screen.tolist(),
'm_from_screen':s.m_from_screen.tolist(),
'gaze_on_srf': s.gaze_on_srf,
'timestamp':frame.timestamp,
'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None
})
if self.running:
self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces))
else:
self.button.status_text = 'tracking paused'
if self.mode == 'Show Markers and Surfaces':
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
def recent_events(self,events):
frame = events.get('frame')
if not frame:
return
self.img_shape = frame.img.shape
self.update_marker_cache()
# self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence)
if s.detected:
events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen.tolist(),'m_from_screen':s.m_from_screen.tolist(),'gaze_on_srf': s.gaze_on_srf, 'timestamp':frame.timestamp,'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None})
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
elif self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence)
self.notify_all({'subject':'surfaces_changed','delay':1})
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def update(self, frame, events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(
frame.index
) # tell precacher that it better have every thing from here on analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
pass
# events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})
if self.mode == "Show marker IDs":
draw_markers(frame.img, self.markers)
# edit surfaces by user
if self.mode == "Surface edit mode":
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
for s, v_idx in self.edit_surfaces:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
s.cache = None
self.heatmap = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def update(self,frame,recent_pupil_positions,events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})
if self.mode.value == 4:
draw_markers(frame.img,self.markers)
# edit surfaces by user
if self.mode.value == 1:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for s,v_idx in self.edit_surfaces:
if s.detected:
pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True)
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
s.cache = None
self.heatmap = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def update(self,frame,events):
self.img_shape = frame.height,frame.width,3
if self.running:
gray = frame.gray
# hack "self.markers" instead "self.screens" is keeped for inheritence compatibility
self.markers = detect_screens(gray)
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d)
if s.detected:
events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.running:
self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces))
else:
self.button.status_text = 'tracking paused'
if self.mode == 'Show Markers and Surfaces':
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get('gaze_positions',[]):
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
p['realtime gaze on ' + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
def gl_display(self):
if self.running:
#apply undistord image to draw
glMatrixMode( GL_MODELVIEW )
make_coord_system_norm_based()
apply_gl_texture(self.frame_img)
if self.show_undistord:
make_coord_system_pixel_based([self.roi[3], self.roi[2], 3])
else:
make_coord_system_pixel_based(self.img_shape)
if self.mode == "Show marker IDs":
visible = [m for m in self.markers if m['visible']]
if self.show_undistord:
draw_markers(self.img_shape, visible, self.roi)
else:
draw_markers(self.img_shape, visible)
if self.mode == "Draw obj" and self.frame_img != None:
if not self.glIsInit :
glInit()
self.glIsInit = True
for m in self.markers:
if m['visible']:
if m['to_draw']:
#drawIn2D( self.camera_intrinsics[0], self.camera_intrinsics[1], m )
if self.show_undistord:
glDrawFromCamera( self.camera_intrinsics[0], self.camera_intrinsics[1], m['rot'], m['trans'], self.img_shape, self.roi, m['obj'] )
else:
glDrawFromCamera( self.camera_intrinsics[0], self.camera_intrinsics[1], m['rot'], m['trans'], self.img_shape, None, m['obj'] )
self.frame_img = None
def update(self,frame,recent_pupil_positions,events):
img = frame.img
self.img_shape = frame.img.shape
if self.robust_detection.value:
self.markers = detect_markers_robust(img,
grid_size = 5,
prev_markers=self.markers,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture.value,
visualize=0,
true_detect_every_frame=3)
else:
self.markers = detect_markers_simple(img,
grid_size = 5,
min_marker_perimeter=self.min_marker_perimeter,
aperture=self.aperture.value,
visualize=0)
# locate surfaces
for s in self.surfaces:
s.locate(self.markers)
if s.detected:
events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.draw_markers.value:
draw_markers(img,self.markers)
# edit surfaces by user
if self.surface_edit_mode:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for s,v_idx in self.edit_surfaces:
if s.detected:
pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True)
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in recent_pupil_positions:
if p['norm_pupil'] is not None:
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze'])))
p['realtime gaze on '+s.name] = gp_on_s
print gp_on_s
#try:
# ser = serial.Serial("/dev/ttyACM0")
# ser.close()
# ser.open()
# print(ser.read())
# s.gaze_on_srf.append(gp_on_s)
#except:
# print("it didn't work!!!!!")
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def recent_events(self, events):
frame = events.get("frame")
if not frame:
return
self.img_shape = frame.height, frame.width, 3
if self.running:
gray = frame.gray
if self.invert_image:
gray = 255 - gray
if self.robust_detection:
self.markers = detect_markers_robust(
gray,
grid_size=5,
aperture=self.aperture,
prev_markers=self.markers,
true_detect_every_frame=3,
min_marker_perimeter=self.min_marker_perimeter,
)
else:
self.markers = detect_markers(
gray,
grid_size=5,
aperture=self.aperture,
min_marker_perimeter=self.min_marker_perimeter,
)
if self.mode == "Show marker IDs":
draw_markers(frame.gray, self.markers)
# locate surfaces, map gaze
for s in self.surfaces:
s.locate(
self.markers,
self.min_marker_perimeter,
self.min_id_confidence,
self.locate_3d,
)
if s.detected:
s.gaze_on_srf = s.map_data_to_surface(events.get("gaze", []),
s.m_from_screen)
s.fixations_on_srf = s.map_data_to_surface(
events.get("fixations", []), s.m_from_screen)
s.update_gaze_history()
else:
s.gaze_on_srf = []
s.fixations_on_srf = []
events["surfaces"] = []
for s in self.surfaces:
if s.detected:
datum = {
"topic":
"surfaces.{}".format(s.name),
"name":
s.name,
"uid":
s.uid,
"m_to_screen":
s.m_to_screen.tolist(),
"m_from_screen":
s.m_from_screen.tolist(),
"gaze_on_srf":
s.gaze_on_srf,
"fixations_on_srf":
s.fixations_on_srf,
"timestamp":
frame.timestamp,
"camera_pose_3d":
s.camera_pose_3d.tolist()
if s.camera_pose_3d is not None else None,
}
events["surfaces"].append(datum)
if self.running:
self.button.status_text = "{}/{}".format(
len([s for s in self.surfaces if s.detected]),
len(self.surfaces))
else:
self.button.status_text = "tracking paused"
if self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
pos = self._last_mouse_pos
for s, v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
def recent_events(self, events):
frame = events.get('frame')
if not frame:
return
self.img_shape = frame.img.shape
self.update_marker_cache()
# self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
self.markers = self.cache[frame.index]
if self.markers is False:
self.markers = []
# tell precacher that it better have every thing from here on analyzed
self.seek_marker_cacher(frame.index)
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers, self.min_marker_perimeter,
self.min_id_confidence)
if s.detected:
events['surfaces'].append({
'name':
s.name,
'uid':
s.uid,
'm_to_screen':
s.m_to_screen.tolist(),
'm_from_screen':
s.m_from_screen.tolist(),
'gaze_on_srf':
s.gaze_on_srf,
'timestamp':
frame.timestamp,
'camera_pose_3d':
s.camera_pose_3d.tolist()
if s.camera_pose_3d is not None else None
})
if self.mode == "Show marker IDs":
draw_markers(frame.img, self.markers)
elif self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
pos = self._last_mouse_pos # inherited from Surface_Tracker
for s, v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache, self.min_marker_perimeter,
self.min_id_confidence)
self.notify_all({
'subject': 'surfaces_changed',
'delay': 1
})
# allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def marker_positions(camera_spec,
videofile,
outfile,
new_camera=None,
start_time=0.0,
end_time=float("inf"),
visualize=False,
output_camera=None,
write=False):
camera = pickle.load(open(camera_spec, 'rb'), encoding='bytes')
image_resolution = camera[b'resolution']
if b'rect_map' not in camera:
camera_matrix = camera[b'camera_matrix']
camera_distortion = camera[b'dist_coefs']
rect_camera_matrix, roi = cv2.getOptimalNewCameraMatrix(
camera_matrix, camera_distortion, image_resolution, 0.0)
rmap = cv2.initUndistortRectifyMap(camera_matrix, camera_distortion,
None, rect_camera_matrix,
image_resolution, cv2.CV_32FC1)
else:
rmap = camera[b'rect_map']
rect_camera_matrix = camera[b'rect_camera_matrix']
K, D, resolution, cm = camera[b'camera_matrix'], camera[
b'dist_coefs'], camera[b'resolution'], rect_camera_matrix
camera = {}
camera['camera_matrix'] = rect_camera_matrix
camera['dist_coefs'] = None
camera['resolution'] = image_resolution
if new_camera is not None:
save_object(camera, new_camera)
#rectify_gaze_data(path, K, D, rect_camera_matrix)
#if new_camera is not None:
# pickle.dump(camera, open(new_camera, 'w'))
video = cv2.VideoCapture(videofile)
video.set(cv2.CAP_PROP_POS_MSEC, start_time * 1000)
frames = []
#marker_tracker = markerdetect.MarkerTracker()
prev_minute = 0.0
marker_cache = []
#fourcc = cv2.VideoWriter_fourcc(*'XVID')
#if write == True:
# out = cv2.VideoWriter(path + "world.mp4",fourcc, 30.0, (1280,720))
while True:
ret, oframe = video.read()
if not ret:
break
frame = cv2.remap(oframe, rmap[0], rmap[1], cv2.INTER_LINEAR)
#if write == True:
# out.write(frame)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
msecs = video.get(cv2.CAP_PROP_POS_MSEC)
time = msecs / 1000.0
if time > end_time:
break
markers = markerdetect.detect_markers(frame,
5,
min_marker_perimeter=15)
marker_cache.append(markers)
frame_d = {
'ts': time,
'markers': markers,
}
frames.append(frame_d)
if not visualize: continue
markerdetect.draw_markers(frame, frame_d['markers'])
cv2.imshow('frameBG', frame)
cv2.waitKey(1)
np.save(outfile, frames)
if write == True:
out.release()
return marker_cache
def update(self, frame, events):
self.img_shape = frame.img.shape
self.update_marker_cache()
# self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(
frame.index
) # tell precacher that it better have every thing from here on analyzed
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers, self.camera_calibration,
self.min_marker_perimeter)
if s.detected:
events['surfaces'].append({
'name': s.name,
'uid': s.uid,
'm_to_screen': s.m_to_screen,
'm_from_screen': s.m_from_screen,
'timestamp': frame.timestamp
})
if self.mode == "Show marker IDs":
draw_markers(frame.img, self.markers)
elif self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
for s, v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None and s not in [
s for s, i in self.edit_surf_verts
]:
s.init_cache(self.cache, self.camera_calibration,
self.min_marker_perimeter)
self.notify_all({
'subject': 'surfaces_changed',
'delay': 1
})
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get('gaze_positions', []):
gp_on_s = tuple(
s.img_to_ref_surface(np.array(p['norm_pos'])))
p['realtime gaze on ' + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def update(self, frame, events):
self.img_shape = frame.height, frame.width, 3
if self.running:
gray = frame.gray
if self.invert_image:
gray = 255 - gray
if self.robust_detection:
self.markers = detect_markers_robust(
gray,
grid_size=5,
aperture=self.aperture,
prev_markers=self.markers,
true_detect_every_frame=3,
min_marker_perimeter=self.min_marker_perimeter,
)
else:
self.markers = detect_markers(
gray, grid_size=5, aperture=self.aperture, min_marker_perimeter=self.min_marker_perimeter
)
if self.mode == "Show marker IDs":
draw_markers(frame.gray, self.markers)
# locate surfaces, map gaze
for s in self.surfaces:
s.locate(
self.markers, self.camera_calibration, self.min_marker_perimeter, self.min_id_confidence, self.locate_3d
)
if s.detected:
s.gaze_on_srf = s.map_data_to_surface(events.get("gaze_positions", []), s.m_from_screen)
else:
s.gaze_on_srf = []
events["surface"] = []
for s in self.surfaces:
if s.detected:
events["surface"].append(
{
"name": s.name,
"uid": s.uid,
"m_to_screen": s.m_to_screen.tolist(),
"m_from_screen": s.m_from_screen.tolist(),
"gaze_on_srf": s.gaze_on_srf,
"timestamp": frame.timestamp,
"camera_pose_3d": s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None,
}
)
if self.running:
self.button.status_text = "%s/%s" % (len([s for s in self.surfaces if s.detected]), len(self.surfaces))
else:
self.button.status_text = "tracking paused"
if self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
for s, v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
