def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
r = p_window_size[0] / 15.0
# compensate for radius of marker
gl.glOrtho(-r, p_window_size[0] + r, p_window_size[1] + r, -r, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
# hacky way of scaling and fitting in different window rations/sizes
grid = _make_grid() * min((p_window_size[0], p_window_size[1] * 5.5 / 4.0))
# center the pattern
grid -= np.mean(grid)
grid += (p_window_size[0] / 2 - r, p_window_size[1] / 2 + r)
draw_points(grid, size=r, color=RGBA(0.0, 0.0, 0.0, 1), sharpness=0.95)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch {} more times to close window.".format(self.clicks_to_close),
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = glfwGetFramebufferSize(
self._window)[0] / glfwGetWindowSize(self._window)[0]
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = map_value(
self.display_pos[0],
out_range=(pad, p_window_size[0] - pad)), map_value(
self.display_pos[1], out_range=(p_window_size[1] - pad, pad))
alpha = 1.0 #interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
r2 = 2 * r
draw_points([screen_pos],
size=60 * r2,
color=RGBA(0., 0., 0., alpha),
sharpness=0.9)
draw_points([screen_pos],
size=38 * r2,
color=RGBA(1., 1., 1., alpha),
sharpness=0.8)
draw_points([screen_pos],
size=19 * r2,
color=RGBA(0., 0., 0., alpha),
sharpness=0.55)
# some feedback on the detection state
color = RGBA(0., .8, 0., alpha) if len(
self.markers) and self.on_position else RGBA(0.8, 0., 0., alpha)
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.))
self.glfont.draw_text(
p_window_size[0] / 2., p_window_size[1] / 4.,
'Touch {} more times to cancel calibration.'.format(
self.clicks_to_close))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self,world_tex):
"""
here we map a selected surface onto a seperate window.
"""
if self._window and self.detected:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_from_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
glOrtho(0, 1, 0, 1,-1,1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
world_tex.draw()
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
# now lets get recent pupil positions on this surface:
for gp in self.gaze_on_srf:
draw_points_norm([gp['norm_pos']],color=RGBA(0.0,0.8,0.5,0.8), size=80)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display(self):
"""
This is where we can draw to any gl surface
by default this is the main window, below we change that
"""
# active our window
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self.window)
# start drawing things:
gl_utils.clear_gl_screen()
# set coordinate system to be between 0 and 1 of the extents of the window
gl_utils.make_coord_system_norm_based()
# draw the image
draw_gl_texture(self.img)
# make coordinte system identical to the img pixel coordinate system
gl_utils.make_coord_system_pixel_based(self.img.shape)
# draw some points on top of the image
# notice how these show up in our window but not in the main window
draw_points([(200, 400), (600, 400)], color=RGBA(0., 4., .8, .8), size=self.my_var)
draw_polyline([(200, 400), (600, 400)], color=RGBA(0., 4., .8, .8), thickness=3)
# since this is our own window we need to swap buffers in the plugin
glfwSwapBuffers(self.window)
# and finally reactive the main window
glfwMakeContextCurrent(active_window)
def _draw_frame(window, path, frames, index: int, show_help: bool):
frames_data = [frame.load(path) for frame in frames[index]]
for frame, frame_meta in zip(frames_data, frames[index]):
PreviewWindow._draw_text(
frame,
"Preview {}/{} (eye{})".format(index + 1, len(frames),
frame_meta.eye_id),
(15, frame.shape[0] - 30),
)
#PreviewWindow._draw_text(
# frame,
# "Confidence: {}".format(frame_meta.confidence),
# (15, frame.shape[0] - 30),
#)
frame = frames_data[0] if len(frames_data) == 1 else np.hstack(
frames_data)
# Present usage hints at first load
if show_help:
PreviewWindow._draw_text(
frame,
"Usage: Use the arrow keys for navigating between frames.",
(15, 40),
)
with PreviewWindow.WindowContextManager(window):
clear_gl_screen()
make_coord_system_norm_based()
draw_gl_texture(frame, interpolation=False)
glfw.glfwSwapBuffers(window)
def gl_display_in_window(self,world_tex_id):
"""
here we map a selected surface onto a seperate window.
"""
if self._window and self.detected:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_from_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
glOrtho(0, 1, 0, 1,-1,1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
draw_named_texture(world_tex_id)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
# now lets get recent pupil positions on this surface:
draw_points_norm(self.gaze_on_srf,color=RGBA(0.,8.,.5,.8), size=80)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
if self.window_should_close:
self.close_window()
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
r = p_window_size[0]/15.
# compensate for radius of marker
gl.glOrtho(-r,p_window_size[0]+r,p_window_size[1]+r,-r ,-1,1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
#hacky way of scaling and fitting in different window rations/sizes
grid = _make_grid()*min((p_window_size[0],p_window_size[1]*5.5/4.))
#center the pattern
grid -= np.mean(grid)
grid +=(p_window_size[0]/2-r,p_window_size[1]/2+r)
draw_points(grid,size=r,color=RGBA(0.,0.,0.,1),sharpness=0.95)
if self.clicks_to_close <5:
self.glfont.set_size(int(p_window_size[0]/30.))
self.glfont.draw_text(p_window_size[0]/2.,p_window_size[1]/4.,'Touch %s more times to close window.'%self.clicks_to_close)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self, img):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# gl stuff that will show on your plugin window goes here
draw_gl_texture(img, interpolation=False)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self,img):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# gl stuff that will show on your plugin window goes here
draw_gl_texture(img,interpolation=False)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# Set Matrix unsing gluOrtho2D to include padding for the marker of radius r
#
############################
# r #
# 0,0##################w,h #
# # # #
# # # #
#r# #r#
# # # #
# # # #
# 0,h##################w,h #
# r #
############################
hdpi_factor = glfwGetFramebufferSize(
self._window)[0] / glfwGetWindowSize(self._window)[0]
r = 110 * self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
# compensate for radius of marker
gl.glOrtho(-r * .6, p_window_size[0] + r * .6,
p_window_size[1] + r * .7, -r * .7, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
screen_pos = denormalize(self.display_pos, p_window_size, flip_y=True)
draw_marker(screen_pos, r, self.pattern_alpha)
#some feedback on the detection state
if self.detected and self.on_position:
cygl_draw_points([screen_pos],
size=5,
color=cygl_rgba(0., 1., 0., self.pattern_alpha),
sharpness=0.95)
else:
cygl_draw_points([screen_pos],
size=5,
color=cygl_rgba(1., 0., 0., self.pattern_alpha),
sharpness=0.95)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.))
self.glfont.draw_text(
p_window_size[0] / 2., p_window_size[1] / 4.,
'Touch %s more times to cancel calibration.' %
self.clicks_to_close)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_frozen_scene(self):
gl_utils.clear_gl_screen()
gl_utils.make_coord_system_norm_based()
self.frozen_scene_tex.draw()
gl_utils.make_coord_system_pixel_based(
(self.g_pool.capture.frame_size[1], self.g_pool.capture.frame_size[0], 3)
)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# gl stuff that will show on your plugin window goes here
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_frozen_scene(self):
gl_utils.clear_gl_screen()
gl_utils.make_coord_system_norm_based()
self.frozen_scene_tex.draw()
gl_utils.make_coord_system_pixel_based(
(self.g_pool.capture.frame_size[1],
self.g_pool.capture.frame_size[0], 3))
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = glfwGetFramebufferSize(
self._window)[0] / glfwGetWindowSize(self._window)[0]
r = 110 * self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = .6 * r
screen_pos = map_value(
self.display_pos[0],
out_range=(pad, p_window_size[0] - pad)), map_value(
self.display_pos[1], out_range=(p_window_size[1] - pad, pad))
alpha = interp_fn(
self.screen_marker_state, 0., 1.,
float(self.sample_duration + self.lead_in + self.lead_out),
float(self.lead_in), float(self.sample_duration + self.lead_in))
draw_concentric_circles(screen_pos, r, 6, alpha)
#some feedback on the detection state
if self.detected and self.on_position:
draw_points([screen_pos],
size=5,
color=RGBA(0., .8, 0., alpha),
sharpness=0.5)
else:
draw_points([screen_pos],
size=5,
color=RGBA(0.8, 0., 0., alpha),
sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.))
self.glfont.draw_text(
p_window_size[0] / 2., p_window_size[1] / 4.,
'Touch %s more times to cancel calibration.' %
self.clicks_to_close)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_t265_window(self):
""" Show new frame in window. """
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self.t265_window)
clear_gl_screen()
if self.last_video_frame is not None:
make_coord_system_norm_based()
draw_gl_texture(self.last_video_frame)
glfwSwapBuffers(self.t265_window)
glfwMakeContextCurrent(active_window)
def consume_events_and_render_buffer():
glfw.make_context_current(main_window)
clear_gl_screen()
if all(c > 0 for c in g_pool.camera_render_size):
glViewport(0, 0, *g_pool.camera_render_size)
for p in g_pool.plugins:
p.gl_display()
glViewport(0, 0, *window_size)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
try:
clipboard = glfw.get_clipboard_string(main_window).decode()
except (AttributeError, glfw.GLFWError):
# clipboard is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.set_clipboard_string(main_window, user_input.clipboard)
for button, action, mods in user_input.buttons:
x, y = glfw.get_cursor_pos(main_window)
pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
main_window, x, y, cached_scale=None
)
pos = normalize(pos, g_pool.camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
# update screen
glfw.swap_buffers(main_window)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = glfwGetFramebufferSize(self._window)[0] / glfwGetWindowSize(self._window)[0]
r = 110 * self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 0.6 * r
screen_pos = (
map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)),
map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)),
)
alpha = interp_fn(
self.screen_marker_state,
0.0,
1.0,
float(self.sample_duration + self.lead_in + self.lead_out),
float(self.lead_in),
float(self.sample_duration + self.lead_in),
)
draw_concentric_circles(screen_pos, r, 6, alpha)
# some feedback on the detection state
if self.detected and self.on_position:
draw_points([screen_pos], size=5, color=RGBA(0.0, 0.8, 0.0, alpha), sharpness=0.5)
else:
draw_points([screen_pos], size=5, color=RGBA(0.8, 0.0, 0.0, alpha), sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch %s more times to cancel calibration." % self.clicks_to_close,
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = glfwGetFramebufferSize(self._window)[0]/glfwGetWindowSize(self._window)[0]
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value,in_range=(0,1),out_range=(0,1)):
ratio = (out_range[1]-out_range[0])/(in_range[1]-in_range[0])
return (value-in_range[0])*ratio+out_range[0]
pad = 90 * r
screen_pos = map_value(self.display_pos[0],out_range=(pad,p_window_size[0]-pad)),map_value(self.display_pos[1],out_range=(p_window_size[1]-pad,pad))
alpha = interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
r2 = 2 * r
draw_points([screen_pos], size=60*r2, color=RGBA(0., 0., 0., alpha), sharpness=0.9)
draw_points([screen_pos], size=38*r2, color=RGBA(1., 1., 1., alpha), sharpness=0.8)
draw_points([screen_pos], size=19*r2, color=RGBA(0., 0., 0., alpha), sharpness=0.55)
# some feedback on the detection state and button pressing
if self.detected and self.on_position and self.space_key_was_pressed:
color = RGBA(.8,.8,0., alpha)
else:
if self.detected:
color = RGBA(0.,.8,0., alpha)
else:
color = RGBA(.8,0.,0., alpha)
draw_points([screen_pos],size=3*r2,color=color,sharpness=0.5)
if self.clicks_to_close <5:
self.glfont.set_size(int(p_window_size[0]/30.))
self.glfont.draw_text(p_window_size[0]/2.,p_window_size[1]/4.,'Touch {} more times to cancel {}.'.format(self.clicks_to_close, self.mode_pretty))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# Set Matrix unsing gluOrtho2D to include padding for the marker of radius r
#
############################
# r #
# 0,0##################w,h #
# # # #
# # # #
#r# #r#
# # # #
# # # #
# 0,h##################w,h #
# r #
############################
hdpi_factor = glfwGetFramebufferSize(self._window)[0]/glfwGetWindowSize(self._window)[0]
r = 110*self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
# compensate for radius of marker
gl.glOrtho(-r*.6,p_window_size[0]+r*.6,p_window_size[1]+r*.7,-r*.7 ,-1,1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
screen_pos = denormalize(self.display_pos,p_window_size,flip_y=True)
alpha = interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
draw_concentric_circles(screen_pos,r,6,alpha)
#some feedback on the detection state
if self.detected and self.on_position:
draw_points([screen_pos],size=5,color=RGBA(0.,1.,0.,alpha),sharpness=0.95)
else:
draw_points([screen_pos],size=5,color=RGBA(1.,0.,0.,alpha),sharpness=0.95)
if self.clicks_to_close <5:
self.glfont.set_size(int(p_window_size[0]/30.))
self.glfont.draw_text(p_window_size[0]/2.,p_window_size[1]/4.,'Touch %s more times to cancel calibration.'%self.clicks_to_close)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
gl.glColor3f(.80, .80, .8)
self.draw_rect(0, 0, 2000, 2000)
self.draw_markers()
hdpi_factor = glfwGetFramebufferSize(self._window)[0]/glfwGetWindowSize(self._window)[0]
r = 110*self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value,in_range=(0,1),out_range=(0,1)):
ratio = (out_range[1]-out_range[0])/(in_range[1]-in_range[0])
return (value-in_range[0])*ratio+out_range[0]
#pad = 0.1*1920
pad = .7*r
screen_pos = map_value(self.display_pos[0],out_range=(pad,p_window_size[0]-pad)),map_value(self.display_pos[1],out_range=(p_window_size[1]-pad,pad))
alpha = interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
if self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out:
draw_concentric_circles(screen_pos,r,6,alpha)
#some feedback on the detection state
if self.detected and self.on_position:
draw_points([screen_pos],size=10*self.marker_scale,color=RGBA(0.,.8,0.,alpha),sharpness=0.5)
elif self.screen_marker_state < self.sample_duration+self.lead_in+self.lead_out:
draw_points([screen_pos],size=10*self.marker_scale,color=RGBA(0.8,0.,0.,alpha),sharpness=0.5)
#if self.clicks_to_close <5:
# self.glfont.set_size(int(p_window_size[0]/30.))
# self.glfont.draw_text(p_window_size[0]/2.,p_window_size[1]/4.,'Touch {} more times to cancel calibration.'.format(self.clicks_to_close))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# Set Matrix unsing gluOrtho2D to include padding for the marker of radius r
#
############################
# r #
# 0,0##################w,h #
# # # #
# # # #
#r# #r#
# # # #
# # # #
# 0,h##################w,h #
# r #
############################
r = 60
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
# compensate for radius of marker
gluOrtho2D(
-r, p_window_size[0] + r, p_window_size[1] + r,
-r) # origin in the top left corner just like the img np-array
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
screen_pos = denormalize(self.display_pos, p_window_size, flip_y=True)
draw_marker(screen_pos)
#some feedback on the detection state
if self.detected and self.on_position:
draw_gl_point(screen_pos, 5, (0., 1., 0., 1.))
else:
draw_gl_point(screen_pos, 5, (1., 0., 0., 1.))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display(self):
"""
This is where we can draw to any gl surface
by default this is the main window, below we change that
"""
# active our window
#active_window = glfwGetCurrentContext()
#glfwMakeContextCurrent(self.window)
# start drawing things:
gl_utils.clear_gl_screen()
# set coordinate system to be between 0 and 1 of the extents of the window
gl_utils.make_coord_system_norm_based()
# draw the image
try:
draw_gl_texture(self.img)
gl_utils.make_coord_system_pixel_based(self.img.shape)
except AttributeError:
pass
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# Set Matrix unsing gluOrtho2D to include padding for the marker of radius r
#
############################
# r #
# 0,0##################w,h #
# # # #
# # # #
#r# #r#
# # # #
# # # #
# 0,h##################w,h #
# r #
############################
r = 60
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetWindowSize(self._window)
# compensate for radius of marker
gluOrtho2D(-r,p_window_size[0]+r,p_window_size[1]+r, -r) # origin in the top left corner just like the img np-array
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
screen_pos = denormalize(self.display_pos,p_window_size,flip_y=True)
draw_marker(screen_pos)
#some feedback on the detection state
if self.detected and self.on_position:
draw_gl_point(screen_pos, 5, (0.,1.,0.,1.))
else:
draw_gl_point(screen_pos, 5, (1.,0.,0.,1.))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def drawing_context(self):
if self.__gl_handle is None:
return
if glfw.glfwWindowShouldClose(self.__gl_handle):
self.close()
return
with self._switch_to_current_context():
clear_gl_screen()
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
gl.glOrtho(0, self.window_size[0], self.window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
yield self.unsafe_handle
glfw.glfwSwapBuffers(self.unsafe_handle)
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
#todo write code to display pattern.
# r = 60.
# gl.glMatrixMode(gl.GL_PROJECTION)
# gl.glLoadIdentity()
# draw_gl_point((-.5,-.5),50.)
# p_window_size = glfwGetWindowSize(self._window)
# # compensate for radius of marker
# x_border,y_border = normalize((r,r),p_window_size)
# # if p_window_size[0]<p_window_size[1]: #taller
# # ratio = p_window_size[1]/float(p_window_size[0])
# # gluOrtho2D(-x_border,1+x_border,y_border, 1-y_border) # origin in the top left corner just like the img np-array
# # else: #wider
# # ratio = p_window_size[0]/float(p_window_size[1])
# # gluOrtho2D(-x_border,ratio+x_border,y_border, 1-y_border) # origin in the top left corner just like the img np-array
# gluOrtho2D(-x_border,1+x_border,y_border, 1-y_border) # origin in the top left corner just like the img np-array
# # Switch back to Model View Matrix
# gl.glMatrixMode(gl.GL_MODELVIEW)
# gl.glLoadIdentity()
# for p in self.display_grid:
# draw_gl_point(p)
# #some feedback on the detection state
# # if self.detected and self.on_position:
# # draw_gl_point(screen_pos, 5.0, (0.,1.,0.,1.))
# # else:
# # draw_gl_point(screen_pos, 5.0, (1.,0.,0.,1.))
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self, surface):
"""
here we map a selected surface onto a seperate window.
"""
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(surface.m_from_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
gluOrtho2D(0, 1, 0, 1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
draw_named_texture(self.g_pool.image_tex)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
#now lets get recent pupil positions on this surface:
try:
gaze_on_surface = [
p['realtime gaze on ' + surface.name]
for p in self.recent_pupil_positions
]
except KeyError:
gaze_on_surface = []
draw_gl_points_norm(gaze_on_surface, color=(0., 8., .5, .8), size=80)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self, world_tex):
"""
here we map a selected surface onto a separate window.
"""
if self._window and self.surface.detected:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self._window)
gl_utils.clear_gl_screen()
# cv uses 3x3 gl uses 4x4 transformation matrices
width, height = self.tracker.camera_model.resolution
img_corners = np.array(
[(0, height), (width, height), (width, 0), (0, 0)], dtype=np.float32
)
denorm_trans = _get_norm_to_points_trans(img_corners)
trans_mat = self.surface.dist_img_to_surf_trans @ denorm_trans
trans_mat = gl_utils.cvmat_to_glmat(trans_mat)
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glPushMatrix()
gl.glLoadIdentity()
gl.glOrtho(0, 1, 0, 1, -1, 1)
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glPushMatrix()
# apply trans_mat to our quad - this will stretch the quad such that the
# surface will span the window extends
gl.glLoadMatrixf(trans_mat)
world_tex.draw()
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glPopMatrix()
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glPopMatrix()
self.draw_recent_pupil_positions()
glfw.glfwSwapBuffers(self._window)
glfw.glfwMakeContextCurrent(active_window)
def gl_display_in_window(self,surface):
"""
here we map a selected surface onto a seperate window.
"""
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(surface.m_from_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
gluOrtho2D(0, 1, 0, 1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
#redraw will use the most recent world img texture and use it again.
redraw_gl_texture( ((0,0),(1,0),(1,1),(0,1)) )
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
#now lets get recent pupil positions on this surface:
try:
gaze_on_surface = [p['realtime gaze on '+surface.name] for p in self.recent_pupil_positions]
except KeyError:
gaze_on_surface = []
draw_gl_points_norm(gaze_on_surface,color=(0.,8.,.5,.8), size=80)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self, world_tex):
"""
here we map a selected surface onto a separate window.
"""
if self._window and self.surface.detected:
active_window = glfw.get_current_context()
glfw.make_context_current(self._window)
gl_utils.clear_gl_screen()
# cv uses 3x3 gl uses 4x4 transformation matrices
width, height = self.tracker.camera_model.resolution
img_corners = np.array(
[(0, height), (width, height), (width, 0), (0, 0)], dtype=np.float32
)
denorm_trans = _get_norm_to_points_trans(img_corners)
trans_mat = self.surface.dist_img_to_surf_trans @ denorm_trans
trans_mat = gl_utils.cvmat_to_glmat(trans_mat)
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glPushMatrix()
gl.glLoadIdentity()
gl.glOrtho(0, 1, 0, 1, -1, 1)
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glPushMatrix()
# apply trans_mat to our quad - this will stretch the quad such that the
# surface will span the window extends
gl.glLoadMatrixf(trans_mat)
world_tex.draw()
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glPopMatrix()
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glPopMatrix()
self.draw_recent_pupil_positions()
glfw.swap_buffers(self._window)
glfw.make_context_current(active_window)
def show_no_rec_window():
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
global rec_dir
def on_drop(window, count, paths):
global rec_dir
rec_dir = paths[0].decode('utf-8')
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if VersionFormat(session_settings.get("version", '0.0')) != get_version(version_file):
logger.info("Session setting are from a different version of this app. I will not use those.")
session_settings.clear()
w, h = session_settings.get('window_size', (1280, 720))
window_pos = session_settings.get('window_position', window_position_default)
glfwWindowHint(GLFW_RESIZABLE, 0)
window = glfwCreateWindow(w, h, 'Pupil Player')
glfwWindowHint(GLFW_RESIZABLE, 1)
glfwMakeContextCurrent(window)
glfwSetWindowPos(window, window_pos[0], window_pos[1])
glfwSetDropCallback(window, on_drop)
glfont = fontstash.Context()
glfont.add_font('roboto', get_roboto_font_path())
glfont.set_align_string(v_align="center", h_align="middle")
glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
gl_utils.basic_gl_setup()
glClearColor(0.5, .5, 0.5, 0.0)
text = 'Drop a recording directory onto this window.'
tip = '(Tip: You can drop a recording directory onto the app icon.)'
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
while not glfwWindowShouldClose(window):
fb_size = glfwGetFramebufferSize(window)
hdpi_factor = float(fb_size[0] / glfwGetWindowSize(window)[0])
gl_utils.adjust_gl_view(*fb_size)
if rec_dir:
if is_pupil_rec_dir(rec_dir):
logger.info("Starting new session with '{}'".format(rec_dir))
text = "Updating recording format."
tip = "This may take a while!"
else:
logger.error("'{}' is not a valid pupil recording".format(rec_dir))
tip = "Oops! That was not a valid recording."
rec_dir = None
gl_utils.clear_gl_screen()
glfont.set_blur(10.5)
glfont.set_color_float((0.0, 0.0, 0.0, 1.))
glfont.set_size(w/25.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text)
glfont.set_size(w/30.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip)
glfont.set_blur(0.96)
glfont.set_color_float((1., 1., 1., 1.))
glfont.set_size(w/25.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text)
glfont.set_size(w/30.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip)
glfwSwapBuffers(window)
if rec_dir:
update_recording_to_recent(rec_dir)
glfwSetWindowShouldClose(window, True)
glfwPollEvents()
session_settings['window_position'] = glfwGetWindowPos(window)
session_settings['version'] = str(get_version(version_file))
session_settings.close()
del glfont
glfwDestroyWindow(window)
def eye(
timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None,
):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
if is_alive_flag.value:
# indicates eye process that this is a duplicated startup
logger.warning("Aborting redundant eye process startup")
return
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
# general imports
import traceback
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible, glViewport
from ui_roi import UIRoi
# monitoring
import psutil
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, AV_Writer
from ndsi import H264Writer
from video_capture import source_classes
from video_capture import manager_classes
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy
pupil_detectors = {
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D,
Detector_Dummy.__name__: Detector_Dummy,
}
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id + 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (600, 90 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
icon_bar_width = 50
window_size = None
camera_render_size = None
hdpi_factor = 1.0
# g_pool holds variables for this process
g_pool = SimpleNamespace()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = "capture"
g_pool.process = "eye{}".format(eye_id)
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x *= hdpi_factor
y *= hdpi_factor
g_pool.gui.update_mouse(x, y)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, g_pool.capture.frame_size)
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
plugins = (g_pool.capture_manager, g_pool.capture)
# call `on_drop` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(p.on_drop(paths) for p in plugins)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
)
if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.capture_manager = None
g_pool.flip = session_settings.get("flip", False)
g_pool.display_mode = session_settings.get("display_mode", "camera_image")
g_pool.display_mode_info_text = {
"camera_image": "Raw eye camera image. This uses the least amount of CPU power",
"roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
"algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
}
capture_manager_settings = session_settings.get(
"capture_manager_settings", ("UVC_Manager", {})
)
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__: c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](
g_pool, **manager_settings
)
if eye_id == 0:
cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"]
else:
cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"]
# Initialize capture
default_settings = (
"UVC_Source",
{"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120},
)
capture_source_settings = overwrite_cap_settings or session_settings.get(
"capture_settings", default_settings
)
source_class_name, source_settings = capture_source_settings
source_class_by_name = {c.__name__: c for c in source_classes}
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings
)
assert g_pool.capture
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get("roi")
if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
pupil_detector_settings = session_settings.get("pupil_detector_settings", None)
last_pupil_detector = pupil_detectors[
session_settings.get("last_pupil_detector", Detector_2D.__name__)
]
g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings)
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_ui()
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# Initialize glfw
glfw.glfwInit()
title = "Pupil Capture - eye {}".format(eye_id)
width, height = g_pool.capture.frame_size
width *= 2
height *= 2
width += icon_bar_width
width, height = session_settings.get("window_size", (width, height))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get("window_position", window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu(
"Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
)
g_pool.iconbar = ui.Scrolling_Menu(
"Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
)
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2],
label="Interface Size",
)
)
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width *= 2
f_height *= 2
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
def uroi_on_mouse_button(button, action, mods):
if g_pool.display_mode == "roi":
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
x, y = glfw.glfwGetCursorPos(main_window)
# pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
x *= hdpi_factor
y *= hdpi_factor
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(
pos, g_pool.capture.frame_size
) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size
):
# if the roi interacts we dont want
# the gui to interact as well
return
general_settings.append(ui.Button("Reset window size", set_window_size))
general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
general_settings.append(
ui.Selector(
"display_mode",
g_pool,
setter=set_display_mode_info,
selection=["camera_image", "roi", "algorithm"],
labels=["Camera Image", "ROI", "Algorithm"],
label="Mode",
)
)
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode]
)
general_settings.append(g_pool.display_mode_info)
detector_selector = ui.Selector(
"pupil_detector",
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Detector_Dummy, Detector_2D, Detector_3D],
labels=["disabled", "C++ 2d detector", "C++ 3d detector"],
label="Detection method",
)
general_settings.append(detector_selector)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
toggle_general_settings(False)
g_pool.pupil_detector.init_ui()
g_pool.capture.init_ui()
g_pool.capture_manager.init_ui()
g_pool.writer = None
def replace_source(source_class_name, source_settings):
g_pool.capture.deinit_ui()
g_pool.capture.cleanup()
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings
)
g_pool.capture.init_ui()
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
g_pool.replace_source = replace_source # for ndsi capture
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 50)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 50)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = "jpeg"
frame_publish_format_recent_warning = False
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning("Process started.")
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification["subject"]
if subject.startswith("eye_process.should_stop"):
if notification["eye_id"] == eye_id:
break
elif subject == "set_detection_mapping_mode":
if notification["mode"] == "3d":
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
elif notification["mode"] == "2d":
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
else:
if not isinstance(g_pool.pupil_detector, Detector_Dummy):
set_detector(Detector_Dummy)
detector_selector.read_only = True
elif subject == "recording.started":
if notification["record_eye"] and g_pool.capture.online:
record_path = notification["rec_path"]
raw_mode = notification["compression"]
logger.info("Will save eye video to: {}".format(record_path))
video_path = os.path.join(
record_path, "eye{}.mp4".format(eye_id)
)
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(
video_path, g_pool.capture.frame_rate
)
elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
g_pool.writer = H264Writer(
video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate,
)
else:
g_pool.writer = AV_Writer(
video_path, g_pool.capture.frame_rate
)
elif subject == "recording.stopped":
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
elif subject.startswith("meta.should_doc"):
ipc_socket.notify(
{
"subject": "meta.doc",
"actor": "eye{}".format(eye_id),
"doc": eye.__doc__,
}
)
elif subject.startswith("frame_publishing.started"):
should_publish_frames = True
frame_publish_format = notification.get("format", "jpeg")
elif subject.startswith("frame_publishing.stopped"):
should_publish_frames = False
frame_publish_format = "jpeg"
elif (
subject.startswith("start_eye_capture")
and notification["target"] == g_pool.process
):
replace_source(notification["name"], notification["args"])
elif notification["subject"].startswith("pupil_detector.set_property"):
target_process = notification.get("target", g_pool.process)
should_apply = target_process == g_pool.process
if should_apply:
try:
property_name = notification["name"]
property_value = notification["value"]
if "2d" in notification["subject"]:
g_pool.pupil_detector.set_2d_detector_property(
property_name, property_value
)
elif "3d" in notification["subject"]:
if not isinstance(g_pool.pupil_detector, Detector_3D):
raise ValueError(
"3d properties are only available"
" if 3d detector is active"
)
g_pool.pupil_detector.set_3d_detector_property(
property_name, property_value
)
else:
raise KeyError(
"Notification subject does not "
"specifiy detector type."
)
logger.debug(
"`{}` property set to {}".format(
property_name, property_value
)
)
except KeyError:
logger.error("Malformed notification received")
logger.debug(traceback.format_exc())
except (ValueError, TypeError):
logger.error("Invalid property or value")
logger.debug(traceback.format_exc())
elif notification["subject"].startswith(
"pupil_detector.broadcast_properties"
):
target_process = notification.get("target", g_pool.process)
should_respond = target_process == g_pool.process
if should_respond:
props = g_pool.pupil_detector.get_detector_properties()
properties_broadcast = {
"subject": "pupil_detector.properties.{}".format(eye_id),
**props, # add properties to broadcast
}
ipc_socket.notify(properties_broadcast)
g_pool.capture.on_notify(notification)
g_pool.capture_manager.on_notify(notification)
# Get an image from the grabber
event = {}
g_pool.capture.recent_events(event)
frame = event.get("frame")
g_pool.capture_manager.recent_events(event)
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
f_height,
f_width,
):
g_pool.pupil_detector.on_resolution_change(
(g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
g_pool.capture.frame_size,
)
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
assert data is not None
except (AttributeError, AssertionError, NameError):
if not frame_publish_format_recent_warning:
frame_publish_format_recent_warning = True
logger.warning(
'{}s are not compatible with format "{}"'.format(
type(frame), frame_publish_format
)
)
else:
frame_publish_format_recent_warning = False
pupil_socket.send(
{
"topic": "frame.eye.{}".format(eye_id),
"width": frame.width,
"height": frame.height,
"index": frame.index,
"timestamp": frame.timestamp,
"format": frame_publish_format,
"__raw_data__": [data],
}
)
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1.0 / dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == "algorithm"
)
if result is not None:
result["id"] = eye_id
result["topic"] = "pupil.{}".format(eye_id)
pupil_socket.send(result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
if frame:
# switch to work in normalized coordinate space
if g_pool.display_mode == "algorithm":
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ("camera_image", "roi"):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
glViewport(0, 0, *camera_render_size)
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
f_width, f_height = g_pool.capture.frame_size
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
if frame and result:
if result["method"] == "3d c++":
eye_ball = result["projected_sphere"]
try:
pts = cv2.ellipse2Poly(
(
int(eye_ball["center"][0]),
int(eye_ball["center"][1]),
),
(
int(eye_ball["axes"][0] / 2),
int(eye_ball["axes"][1] / 2),
),
int(eye_ball["angle"]),
0,
360,
8,
)
except ValueError as e:
pass
else:
draw_polyline(
pts,
2,
RGBA(0.0, 0.9, 0.1, result["model_confidence"]),
)
if result["confidence"] > 0:
if "ellipse" in result:
pts = cv2.ellipse2Poly(
(
int(result["ellipse"]["center"][0]),
int(result["ellipse"]["center"][1]),
),
(
int(result["ellipse"]["axes"][0] / 2),
int(result["ellipse"]["axes"][1] / 2),
),
int(result["ellipse"]["angle"]),
0,
360,
15,
)
confidence = result["confidence"] * 0.7
draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence))
draw_points(
[result["ellipse"]["center"]],
size=20,
color=RGBA(1.0, 0.0, 0.0, confidence),
sharpness=1.0,
)
glViewport(0, 0, *camera_render_size)
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == "roi":
g_pool.u_r.draw_points(g_pool.gui.scale)
glViewport(0, 0, *window_size)
make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
unused_elements = g_pool.gui.update()
for butt in unused_elements.buttons:
uroi_on_mouse_button(*butt)
make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
g_pool.pupil_detector.visualize() # detector decides if we visualize or not
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer = None
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
# save session persistent settings
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["roi"] = g_pool.u_r.get()
session_settings["flip"] = g_pool.flip
session_settings["display_mode"] = g_pool.display_mode
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["capture_settings"] = (
g_pool.capture.class_name,
g_pool.capture.get_init_dict(),
)
session_settings["capture_manager_settings"] = (
g_pool.capture_manager.class_name,
g_pool.capture_manager.get_init_dict(),
)
session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
session_settings["version"] = str(g_pool.version)
session_settings[
"last_pupil_detector"
] = g_pool.pupil_detector.__class__.__name__
session_settings[
"pupil_detector_settings"
] = g_pool.pupil_detector.get_settings()
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
g_pool.capture.deinit_ui()
g_pool.capture_manager.deinit_ui()
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
glfw.glfwDestroyWindow(main_window)
g_pool.gui.terminate()
glfw.glfwTerminate()
logger.info("Process shutting down.")
def gl_display_in_window(self):
active_window = glfwGetCurrentContext()
if glfwWindowShouldClose(self._window):
self.close_window()
return
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = getHDPIFactor(self._window)
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
p_window_size = glfwGetFramebufferSize(self._window)
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = (
map_value(self.display_pos[0], out_range=(pad, p_window_size[0] - pad)),
map_value(self.display_pos[1], out_range=(p_window_size[1] - pad, pad)),
)
alpha = (
1.0
) # interp_fn(self.screen_marker_state,0.,1.,float(self.sample_duration+self.lead_in+self.lead_out),float(self.lead_in),float(self.sample_duration+self.lead_in))
r2 = 2 * r
draw_points(
[screen_pos], size=60 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.9
)
draw_points(
[screen_pos], size=38 * r2, color=RGBA(1.0, 1.0, 1.0, alpha), sharpness=0.8
)
draw_points(
[screen_pos], size=19 * r2, color=RGBA(0.0, 0.0, 0.0, alpha), sharpness=0.55
)
# some feedback on the detection state
color = (
RGBA(0.0, 0.8, 0.0, alpha)
if len(self.markers) and self.on_position
else RGBA(0.8, 0.0, 0.0, alpha)
)
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch {} more times to cancel calibration.".format(
self.clicks_to_close
),
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def player(g_pool,size):
"""player
- Shows 9 point calibration pattern
- Plays a source video synchronized with world process
- Get src videos from directory (glob)
- Iterate through videos on each record event
"""
grid = make_grid()
grid *=2.5###scale to fit
# player object
player = Temp()
player.play_list = glob('src_video/*')
path_parent = os.path.dirname( os.path.abspath(sys.argv[0]))
player.playlist = [os.path.join(path_parent, path) for path in player.play_list]
player.captures = [autoCreateCapture(src) for src in player.playlist]
print "Player found %i videos in src_video"%len(player.captures)
player.captures = [c for c in player.captures if c is not None]
print "Player sucessfully loaded %i videos in src_video"%len(player.captures)
# for c in player.captures: c.auto_rewind = False
player.current_video = 0
# Callbacks
def on_resize(w, h):
adjust_gl_view(w,h)
def on_key(key, pressed):
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if char == ord('9'):
g_pool.cal9.value = True
g_pool.calibrate.value = True
def on_close():
g_pool.quit.value = True
print "Player Process closing from window"
# initialize glfw
glfwInit()
glfwOpenWindow(size[0], size[1], 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("Player")
glfwSetWindowPos(100,0)
glfwDisable(GLFW_AUTO_POLL_EVENTS)
#Callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
#gl state settings
gl.glEnable( gl.GL_BLEND )
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glClearColor(1.,1.,1.,0.)
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
glfwPollEvents()
if g_pool.player_refresh.wait(0.01):
g_pool.player_refresh.clear()
clear_gl_screen()
if g_pool.cal9.value:
circle_id,step = g_pool.cal9_circle_id.value,g_pool.cal9_step.value
gl.glColor4f(0.0,0.0,0.0,1.0)
gl.glPointSize(40)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glBegin(gl.GL_POINTS)
for p in grid:
gl.glVertex3f(p[0],p[1],0.0)
gl.glEnd()
###display the animated target dot
gl.glPointSize((40)*(1.01-(step+1)/80.0))
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ZERO)
if g_pool.ref_x.value or g_pool.ref_y.value: ###if pattern detected
gl.glColor4f(0.0,0.5,0.0,1.0)
else:
gl.glColor4f(0.5,0.0,0.0,1.0)
gl.glBegin(gl.GL_POINTS)
gl.glVertex3f(grid[circle_id][0],grid[circle_id][1],0.0)
gl.glEnd()
elif g_pool.play.value:
s, img = player.captures[player.current_video].read_RGB()
if s:
draw_gl_texture(image)
else:
player.captures[player.current_video].rewind()
player.current_video +=1
if player.current_video >= len(player.captures):
player.current_video = 0
g_pool.play.value = False
glfwSwapBuffers()
glfwCloseWindow()
glfwTerminate()
print "PLAYER Process closed"
def show_no_rec_window():
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
global rec_dir
def on_drop(window, count, paths):
global rec_dir
rec_dir = paths[0].decode('utf-8')
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if VersionFormat(session_settings.get("version", '0.0')) != get_version(version_file):
logger.info("Session setting are from a different version of this app. I will not use those.")
session_settings.clear()
w, h = session_settings.get('window_size', (1280, 720))
window_pos = session_settings.get('window_position', window_position_default)
glfwWindowHint(GLFW_RESIZABLE, 0)
window = glfwCreateWindow(w, h, 'Pupil Player')
glfwWindowHint(GLFW_RESIZABLE, 1)
glfwMakeContextCurrent(window)
glfwSetWindowPos(window, window_pos[0], window_pos[1])
glfwSetDropCallback(window, on_drop)
glfont = fontstash.Context()
glfont.add_font('roboto', get_roboto_font_path())
glfont.set_align_string(v_align="center", h_align="middle")
glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
gl_utils.basic_gl_setup()
glClearColor(0.5, .5, 0.5, 0.0)
text = 'Drop a recording directory onto this window.'
tip = '(Tip: You can drop a recording directory onto the app icon.)'
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
while not glfwWindowShouldClose(window):
fb_size = glfwGetFramebufferSize(window)
hdpi_factor = float(fb_size[0] / glfwGetWindowSize(window)[0])
gl_utils.adjust_gl_view(*fb_size)
if rec_dir:
if is_pupil_rec_dir(rec_dir):
logger.info("Starting new session with '{}'".format(rec_dir))
text = "Updating recording format."
tip = "This may take a while!"
else:
logger.error("'{}' is not a valid pupil recording".format(rec_dir))
tip = "Oops! That was not a valid recording."
rec_dir = None
gl_utils.clear_gl_screen()
glfont.set_blur(10.5)
glfont.set_color_float((0.0, 0.0, 0.0, 1.))
glfont.set_size(w/25.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text)
glfont.set_size(w/30.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip)
glfont.set_blur(0.96)
glfont.set_color_float((1., 1., 1., 1.))
glfont.set_size(w/25.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .3*h*hdpi_factor, text)
glfont.set_size(w/30.*hdpi_factor)
glfont.draw_text(w/2*hdpi_factor, .4*h*hdpi_factor, tip)
glfwSwapBuffers(window)
if rec_dir:
update_recording_to_recent(rec_dir)
glfwSetWindowShouldClose(window, True)
glfwPollEvents()
session_settings['window_position'] = glfwGetWindowPos(window)
session_settings['version'] = str(get_version(version_file))
session_settings.close()
del glfont
glfwDestroyWindow(window)
def show_no_rec_window():
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'" % new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'%s' is not a valid pupil recording" %
new_rec_dir)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version",
VersionFormat('0.0')) < get_version(version_file):
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
w, h = session_settings.get('window_size', (1280, 720))
window_pos = session_settings.get('window_position', (0, 0))
glfwWindowHint(GLFW_RESIZABLE, 0)
window = glfwCreateWindow(w, h, 'Pupil Player')
glfwWindowHint(GLFW_RESIZABLE, 1)
glfwMakeContextCurrent(window)
glfwSetWindowPos(window, window_pos[0], window_pos[1])
glfwSetDropCallback(window, on_drop)
adjust_gl_view(w, h)
glfont = fontstash.Context()
glfont.add_font('roboto', get_roboto_font_path())
glfont.set_align_string(v_align="center", h_align="middle")
glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
basic_gl_setup()
glClearColor(0.5, .5, 0.5, 0.0)
text = 'Drop a recording directory onto this window.'
tip = '(Tip: You can drop a recording directory onto the app icon.)'
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
while not glfwWindowShouldClose(window):
clear_gl_screen()
glfont.set_blur(10.5)
glfont.set_color_float((0.0, 0.0, 0.0, 1.))
glfont.set_size(w / 25.)
glfont.draw_text(w / 2, .3 * h, text)
glfont.set_size(w / 30.)
glfont.draw_text(w / 2, .4 * h, tip)
glfont.set_blur(0.96)
glfont.set_color_float((1., 1., 1., 1.))
glfont.set_size(w / 25.)
glfont.draw_text(w / 2, .3 * h, text)
glfont.set_size(w / 30.)
glfont.draw_text(w / 2, .4 * h, tip)
glfwSwapBuffers(window)
glfwPollEvents()
session_settings['window_position'] = glfwGetWindowPos(window)
session_settings['version'] = get_version(version_file)
session_settings.close()
del glfont
glfwDestroyWindow(window)
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,
'eye%s.log' % eye_id),
mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'Eye' + str(eye_id) +
' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'EYE' + str(eye_id) +
' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == GLFW_RELEASE and u_r.active_edit_pt:
u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos, (frame.width, frame.height)) # Position in img pixels
if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if u_r.active_edit_pt:
pos = normalize((x, y), glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
u_r.move_vertex(u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': cap_size, 'frame_rate': 30}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
pipe_to_world.send('eye%s process ready' % eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
}
# g_pool.draw_pupil = session_settings.get('draw_pupil',True)
u_r = UIRoi(frame.img.shape)
u_r.set(session_settings.get('roi', u_r.get()))
writer = None
pupil_detector = Canny_Detector(g_pool)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
# Initialize glfw
glfwInit()
if g_pool.binocular:
title = "Binocular eye %s" % eye_id
else:
title = 'Eye'
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl_init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# let detector add its GUI
pupil_detector.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetWindowIconifyCallback(main_window, on_iconify)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1 / 60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if pipe_to_world.poll():
command, raw_mode = pipe_to_world.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(record_path,
"eye%s_timestamps.npy" % eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path, cap.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = AV_Writer(video_path, cap.frame_rate)
timestamps = []
else:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(
frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
# switch to work in pixel space
make_coord_system_pixel_based((frame.height, frame.width, 3),
g_pool.flip)
if result['confidence'] > 0:
if result.has_key('axes'):
pts = cv2.ellipse2Poly((int(
result['center'][0]), int(result['center'][1])),
(int(result['axes'][0] / 2),
int(result['axes'][1] / 2)),
int(result['angle']), 0, 360,
15)
cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
cygl_draw_points([result['center']],
size=20,
color=cygl_rgba(1., 0., 0., .5),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
u_r.draw(g_pool.gui.scale)
#update screen
glfwSwapBuffers(main_window)
glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
pupil_detector.cleanup()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
#flushing queue in case world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def eye(g_pool, cap_src, cap_size):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye.log'),
mode='w')
fh.setLevel(logging.INFO)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.WARNING)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'E Y E Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# Callback functions
def on_resize(window, w, h):
adjust_gl_view(w, h)
atb.TwWindowSize(w, h)
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key, int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
def on_char(window, char):
if not atb.TwEventCharGLFW(char, 1):
pass
def on_button(window, button, action, mods):
if not atb.TwEventMouseButtonGLFW(button, int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos, (frame.img.shape[1],
frame.img.shape[0])) # pos in frame.img pixels
u_r.setStart(pos)
bar.draw_roi.value = 1
else:
bar.draw_roi.value = 0
def on_pos(window, x, y):
if atb.TwMouseMotion(int(x), int(y)):
pass
if bar.draw_roi.value == 1:
pos = x, y
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # pos in frame.img pixels
u_r.setEnd(pos)
def on_scroll(window, x, y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# Helper functions called by the main atb bar
def start_roi():
bar.display.value = 1
bar.draw_roi.value = 2
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1. / dt - bar.fps.value)
bar.dt.value = dt
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
# load session persistent settings
session_settings = shelve.open(os.path.join(g_pool.user_dir,
'user_settings_eye'),
protocol=2)
def load(var_name, default):
return session_settings.get(var_name, default)
def save(var_name, var):
session_settings[var_name] = var
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size)
if cap is None:
logger.error("Did not receive valid Capture")
return
# check if it works
frame = cap.get_frame()
if frame.img is None:
logger.error("Could not retrieve image from capture")
cap.close()
return
height, width = frame.img.shape[:2]
cap.auto_rewind = False
u_r = Roi(frame.img.shape)
u_r.set(load('roi', default=None))
writer = None
pupil_detector = Canny_Detector(g_pool)
atb.init()
# Create main ATB Controls
bar = atb.Bar(name="Eye",
label="Display",
help="Scene controls",
color=(50, 50, 50),
alpha=100,
text='light',
position=(10, 10),
refresh=.3,
size=(200, 100))
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.dt = c_float(0.0)
bar.sleep = c_float(0.0)
bar.display = c_int(load('bar.display', 0))
bar.draw_pupil = c_bool(load('bar.draw_pupil', True))
bar.draw_roi = c_int(0)
dispay_mode_enum = atb.enum("Mode", {
"Camera Image": 0,
"Region of Interest": 1,
"Algorithm": 2
})
bar.add_var("FPS", bar.fps, step=1., readonly=True)
bar.add_var("Mode",
bar.display,
vtype=dispay_mode_enum,
help="select the view-mode")
bar.add_var("Show_Pupil_Point", bar.draw_pupil)
bar.add_button("Draw_ROI",
start_roi,
help="drag on screen to select a region of interest")
bar.add_var("SlowDown", bar.sleep, step=0.01, min=0.0)
bar.add_var("SaveSettings&Exit", g_pool.quit)
cap.create_atb_bar(pos=(220, 10))
# create a bar for the detector
pupil_detector.create_atb_bar(pos=(10, 120))
glfwInit()
window = glfwCreateWindow(width, height, "Eye", None, None)
glfwMakeContextCurrent(window)
# Register callbacks window
glfwSetWindowSizeCallback(window, on_resize)
glfwSetWindowCloseCallback(window, on_close)
glfwSetKeyCallback(window, on_key)
glfwSetCharCallback(window, on_char)
glfwSetMouseButtonCallback(window, on_button)
glfwSetCursorPosCallback(window, on_pos)
glfwSetScrollCallback(window, on_scroll)
glfwSetWindowPos(window, 800, 0)
on_resize(window, width, height)
# gl_state settings
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
# event loop
while not g_pool.quit.value:
frame = cap.get_frame()
if frame.img is None:
break
update_fps()
sleep(bar.sleep.value) # for debugging only
if pupil_detector.should_sleep:
sleep(16)
pupil_detector.should_sleep = False
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if g_pool.eye_rx.poll():
command = g_pool.eye_rx.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %(record_path)s")
video_path = os.path.join(record_path, "eye.avi")
timestamps_path = os.path.join(record_path,
"eye_timestamps.npy")
writer = cv2.VideoWriter(
video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value,
(frame.img.shape[1], frame.img.shape[0]))
timestamps = []
else:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write(frame.img)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(frame,
user_roi=u_r,
visualize=bar.display.value == 2)
# stream the result
g_pool.pupil_queue.put(result)
# VISUALIZATION direct visualizations on the frame.img data
if bar.display.value == 1:
# and a solid (white) frame around the user defined ROI
r_img = frame.img[u_r.lY:u_r.uY, u_r.lX:u_r.uX]
r_img[:, 0] = 255, 255, 255
r_img[:, -1] = 255, 255, 255
r_img[0, :] = 255, 255, 255
r_img[-1, :] = 255, 255, 255
# GL-drawing
clear_gl_screen()
draw_gl_texture(frame.img)
if result['norm_pupil'] is not None and bar.draw_pupil.value:
if result.has_key('axes'):
pts = cv2.ellipse2Poly(
(int(result['center'][0]), int(result['center'][1])),
(int(result["axes"][0] / 2), int(result["axes"][1] / 2)),
int(result["angle"]), 0, 360, 15)
draw_gl_polyline(pts, (1., 0, 0, .5))
draw_gl_point_norm(result['norm_pupil'], color=(1., 0., 0., 0.5))
atb.draw()
glfwSwapBuffers(window)
glfwPollEvents()
# END while running
# in case eye reconding was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
# save session persistent settings
save('roi', u_r.get())
save('bar.display', bar.display.value)
save('bar.draw_pupil', bar.draw_pupil.value)
session_settings.close()
pupil_detector.cleanup()
cap.close()
atb.terminate()
glfwDestroyWindow(window)
glfwTerminate()
#flushing queue incase world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def world(g_pool):
"""world
"""
###Callback funtions
def on_resize(w, h):
atb.TwWindowSize(w, h);
adjust_gl_view(w,h)
def on_key(key, pressed):
if not atb.TwEventKeyboardGLFW(key,pressed):
if pressed:
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if not atb.TwEventCharGLFW(char,pressed):
pass
def on_button(button, pressed):
if not atb.TwEventMouseButtonGLFW(button,pressed):
if pressed:
pos = glfwGetMousePos()
pos = normalize(pos,glfwGetWindowSize())
pos = denormalize(pos,(img.shape[1],img.shape[0]) ) #pos in img pixels
ref.detector.new_ref(pos)
def on_pos(x, y):
if atb.TwMouseMotion(x,y):
pass
def on_scroll(pos):
if not atb.TwMouseWheel(pos):
pass
def on_close():
g_pool.quit.value = True
print "WORLD Process closing from window"
ref = Temp()
ref.detector = no_Detector(g_pool.calibrate,g_pool.ref_x,g_pool.ref_y)
###objects as variable containers
# pattern object
pattern = Temp()
pattern.centers = None
pattern.obj_grid = gen_pattern_grid((4, 11)) # calib grid
pattern.obj_points = []
pattern.img_points = []
pattern.map = (0, 2, 7, 16, 21, 23, 39, 40, 42)
pattern.board_centers = None
# gaze object
gaze = Temp()
gaze.map_coords = (0., 0.)
gaze.image_coords = (0., 0.)
# record object
record = Temp()
record.writer = None
record.path_parent = os.path.dirname(os.path.abspath(sys.argv[0]))
record.path = None
record.counter = 0
# initialize capture, check if it works
cap = autoCreateCapture(g_pool.world_src, g_pool.world_size)
if cap is None:
print "WORLD: Error could not create Capture"
return
s, img = cap.read_RGB()
if not s:
print "WORLD: Error could not get image"
return
height,width = img.shape[:2]
###helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1 / dt - bar.fps.value)
def set_window_size(mode,data):
height,width = img.shape[:2]
ratio = (1,.75,.5,.25)[mode]
w,h = int(width*ratio),int(height*ratio)
glfwSetWindowSize(w,h)
data.value=mode #update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def start_calibration():
c_type = bar.calibration_type.value
if c_type == cal_type["Directed 9-Point"]:
print 'WORLD: Starting Directed 9-Point calibration.'
ref.detector = Nine_Point_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y,
shared_stage=g_pool.cal9_stage,
shared_step=g_pool.cal9_step,
shared_cal9_active=g_pool.cal9,
shared_circle_id=g_pool.cal9_circle_id,
auto_advance=False)
elif c_type == cal_type["Automated 9-Point"]:
print 'WORLD: Starting Automated 9-Point calibration.'
ref.detector = Nine_Point_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y,
shared_stage=g_pool.cal9_stage,
shared_step=g_pool.cal9_step,
shared_cal9_active=g_pool.cal9,
shared_circle_id=g_pool.cal9_circle_id,
auto_advance=True)
elif c_type == cal_type["Natural Features"]:
print 'WORLD: Starting Natural Features calibration.'
ref.detector = Natural_Features_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y)
elif c_type == cal_type["Black Dot"]:
print 'WORLD: Starting Black Dot calibration.'
ref.detector = Black_Dot_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y)
def advance_calibration():
ref.detector.advance()
def stop_calibration():
ref.detector = no_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y)
### Initialize ant tweak bar inherits from atb.Bar
atb.init()
bar = atb.Bar(name = "World", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,
text='light', position=(10, 10),refresh=.3, size=(200, 200))
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.calibration_type = c_int(1)
bar.show_calib_result = c_bool(0)
bar.calibration_images = False
bar.record_video = c_bool(0)
bar.record_running = c_bool(0)
bar.play = g_pool.play
bar.window_size = c_int(0)
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
cal_type = {"Directed 9-Point":0,"Automated 9-Point":1,"Natural Features":3,"Black Dot":4}#"Manual 9-Point":2
calibrate_type_enum = atb.enum("Calibration Method",cal_type)
bar.rec_name = create_string_buffer(512)
# play and record can be tied together via pointers to the objects
# bar.play = bar.record_video
bar.add_var("FPS", bar.fps, step=1., readonly=True)
bar.add_var("Display_Size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
bar.add_var("Cal/Calibration_Method",bar.calibration_type, vtype=calibrate_type_enum)
bar.add_button("Cal/Start_Calibration",start_calibration, key='c')
bar.add_button("Cal/Next_Point",advance_calibration,key="SPACE", help="Hit space to calibrate on next dot")
bar.add_button("Cal/Stop_Calibration",stop_calibration, key='d')
bar.add_var("Cal/show_calibration_result",bar.show_calib_result, help="yellow lines indecate fit error, red outline shows the calibrated area.")
bar.add_var("Rec/rec_name",bar.rec_name)
bar.add_var("Rec/Record_Video", bar.record_video, key="r", help="Start/Stop Recording")
bar.add_separator("Sep1")
bar.add_var("Play Source Video", bar.play)
bar.add_var("Exit", g_pool.quit)
#add 4vl2 camera controls to a seperate ATB bar
if cap.controls is not None:
c_bar = atb.Bar(name="Camera_Controls", label=cap.name,
help="UVC Camera Controls", color=(50,50,50), alpha=100,
text='light',position=(220, 10),refresh=2., size=(200, 200))
# c_bar.add_var("auto_refresher",vtype=atb.TW_TYPE_BOOL8,getter=cap.uvc_refresh_all,setter=None,readonly=True)
# c_bar.define(definition='visible=0', varname="auto_refresher")
sorted_controls = [c for c in cap.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type=="bool":
c_bar.add_var(name,vtype=atb.TW_TYPE_BOOL8,getter=control.get_val,setter=control.set_val)
elif control.type=='int':
c_bar.add_var(name,vtype=atb.TW_TYPE_INT32,getter=control.get_val,setter=control.set_val)
c_bar.define(definition='min='+str(control.min), varname=name)
c_bar.define(definition='max='+str(control.max), varname=name)
c_bar.define(definition='step='+str(control.step), varname=name)
elif control.type=="menu":
if control.menu is None:
vtype = None
else:
vtype= atb.enum(name,control.menu)
c_bar.add_var(name,vtype=vtype,getter=control.get_val,setter=control.set_val)
if control.menu is None:
c_bar.define(definition='min='+str(control.min), varname=name)
c_bar.define(definition='max='+str(control.max), varname=name)
c_bar.define(definition='step='+str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
# c_bar.define(definition='readonly=1',varname=control.name)
c_bar.add_button("refresh",cap.update_from_device)
c_bar.add_button("load defaults",cap.load_defaults)
else:
c_bar = None
### Initialize glfw
glfwInit()
height,width = img.shape[:2]
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("World")
glfwSetWindowPos(0,0)
#register callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
glfwSetMouseButtonCallback(on_button)
glfwSetMousePosCallback(on_pos)
glfwSetMouseWheelCallback(on_scroll)
#gl_state settings
import OpenGL.GL as gl
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glPointSize(20)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
###event loop
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
update_fps()
# get an image from the grabber
s, img = cap.read()
ref.detector.detect(img)
if ref.detector.is_done():
stop_calibration()
g_pool.player_refresh.set()
# #gather pattern centers and find cam intrisics
# if bar.screen_shot and pattern.centers is not None:
# bar.screen_shot = False
# # calibrate the camera intrinsics if the board is found
# # append list of circle grid center points to pattern.img_points
# # append generic list of circle grid pattern type to pattern.obj_points
# pattern.centers = circle_grid(img)
# pattern.img_points.append(pattern.centers)
# pattern.obj_points.append(pattern.obj_grid)
# print "Number of Patterns Captured:", len(pattern.img_points)
# #if pattern.img_points.shape[0] > 10:
# if len(pattern.img_points) > 10:
# camera_matrix, dist_coefs = calibrate_camera(np.asarray(pattern.img_points),
# np.asarray(pattern.obj_points),
# (img.shape[1], img.shape[0]))
# np.save("camera_matrix.npy", camera_matrix)
# np.save("dist_coefs.npy", dist_coefs)
# pattern.img_points = []
# bar.find_pattern.value = False
### Setup recording process
if bar.record_video and not bar.record_running:
record.path = os.path.join(record.path_parent, "data%03d/" % record.counter)
while True:
try:
os.mkdir(record.path)
break
except:
print "We dont want to overwrite data, incrementing counter & trying to make new data folder"
record.counter += 1
record.path = os.path.join(record.path_parent, "data%03d/" % record.counter)
#video
video_path = os.path.join(record.path, "world.avi")
#FFV1 -- good speed lossless big file
#DIVX -- good speed good compression medium file
record.writer = cv2.VideoWriter(video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (img.shape[1], img.shape[0]))
# positions data to eye process
g_pool.pos_record.value = True
g_pool.eye_tx.send(record.path)
bar.record_running = 1
g_pool.frame_count_record.value = 0
# While Recording...
if bar.record_video and bar.record_running:
# Save image frames to video writer
# increment the frame_count_record value
# Eye positions can be associated with frames of recording even if different framerates are used
record.writer.write(img)
g_pool.frame_count_record.value += 1
# Finish all recordings, clean up.
if not bar.record_video and bar.record_running:
# for conviniece: copy camera intrinsics into each data folder at the end of a recording.
try:
camera_matrix = np.load("camera_matrix.npy")
dist_coefs = np.load("dist_coefs.npy")
cam_path = os.path.join(record.path, "camera_matrix.npy")
dist_path = os.path.join(record.path, "dist_coefs.npy")
np.save(cam_path, camera_matrix)
np.save(dist_path, dist_coefs)
except:
print "no camera intrinsics found, will not copy them into data folder"
g_pool.pos_record.value = 0
del record.writer
bar.record_running = 0
###render the screen
clear_gl_screen()
cv2.cvtColor(img, cv2.COLOR_BGR2RGB,img)
draw_gl_texture(img)
###render calibration results:
if bar.show_calib_result.value:
cal_pt_cloud = np.load("cal_pt_cloud.npy")
pX,pY,wX,wY = cal_pt_cloud.transpose()
map_fn = get_map_from_cloud(cal_pt_cloud,(width,height))
modelled_world_pts = map_fn((pX,pY))
pts = np.array(modelled_world_pts,dtype=np.float32).transpose()
calib_bounds = cv2.convexHull(pts)[:,0]
for observed,modelled in zip(zip(wX,wY),np.array(modelled_world_pts).transpose()):
draw_gl_polyline_norm((modelled,observed),(1.,0.5,0.,.5))
draw_gl_polyline_norm(calib_bounds,(1.0,0,0,.5))
#render visual feedback from detector
ref.detector.display(img)
# render detector point
if ref.detector.pos[0] or ref.detector.pos[1]:
draw_gl_point_norm(ref.detector.pos,(0.,1.,0.,0.5))
# update gaze point from shared variable pool and draw on screen. If both coords are 0: no pupil pos was detected.
if g_pool.gaze_x.value !=0 or g_pool.gaze_y.value !=0:
draw_gl_point_norm((g_pool.gaze_x.value, g_pool.gaze_y.value),(1.,0.,0.,0.5))
atb.draw()
glfwSwapBuffers()
###end while running clean-up
print "WORLD Process closed"
glfwCloseWindow()
glfwTerminate()
def gl_display_in_window(self):
if glfwWindowShouldClose(self._window):
self.stop()
return
p_window_size = glfwGetFramebufferSize(self._window)
if p_window_size == (0, 0):
# On Windows we get a window_size of (0, 0) when either minimizing the
# Window or when tabbing out (rendered only in the background). We get
# errors when we call the code below with window size (0, 0). Anyways we
# probably want to stop calibration in this case as it will screw up the
# calibration anyways.
self.stop()
return
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
hdpi_factor = getHDPIFactor(self._window)
r = self.marker_scale * hdpi_factor
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
gl.glOrtho(0, p_window_size[0], p_window_size[1], 0, -1, 1)
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
def map_value(value, in_range=(0, 1), out_range=(0, 1)):
ratio = (out_range[1] - out_range[0]) / (in_range[1] - in_range[0])
return (value - in_range[0]) * ratio + out_range[0]
pad = 90 * r
screen_pos = (
map_value(self.display_pos[0],
out_range=(pad, p_window_size[0] - pad)),
map_value(self.display_pos[1],
out_range=(p_window_size[1] - pad, pad)),
)
alpha = interp_fn(
self.screen_marker_state,
0.0,
1.0,
float(self.sample_duration + self.lead_in + self.lead_out),
float(self.lead_in),
float(self.sample_duration + self.lead_in),
)
r2 = 2 * r
draw_points([screen_pos],
size=60 * r2,
color=RGBA(0.0, 0.0, 0.0, alpha),
sharpness=0.9)
draw_points([screen_pos],
size=38 * r2,
color=RGBA(1.0, 1.0, 1.0, alpha),
sharpness=0.8)
draw_points([screen_pos],
size=19 * r2,
color=RGBA(0.0, 0.0, 0.0, alpha),
sharpness=0.55)
# some feedback on the detection state
color = (RGBA(0.0, 0.8, 0.0, alpha) if len(self.markers)
and self.on_position else RGBA(0.8, 0.0, 0.0, alpha))
draw_points([screen_pos], size=3 * r2, color=color, sharpness=0.5)
if self.clicks_to_close < 5:
self.glfont.set_size(int(p_window_size[0] / 30.0))
self.glfont.draw_text(
p_window_size[0] / 2.0,
p_window_size[1] / 4.0,
"Touch {} more times to cancel {}.".format(
self.clicks_to_close, self.mode_pretty),
)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def eye(
timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None,
hide_ui=False,
):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
``start_eye_plugin``: Start plugins in eye process
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
if is_alive_flag.value:
# indicates eye process that this is a duplicated startup
logger.warning("Aborting redundant eye process startup")
return
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
# general imports
import traceback
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible, glViewport
from ui_roi import UIRoi
# monitoring
import psutil
# Plug-ins
from plugin import Plugin_List
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, MPEG_Writer
from ndsi import H264Writer
from video_capture import source_classes, manager_classes
from background_helper import IPC_Logging_Task_Proxy
from pupil_detector_plugins import available_detector_plugins
from pupil_detector_plugins.manager import PupilDetectorManager
IPC_Logging_Task_Proxy.push_url = ipc_push_url
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
# NOTE: Interrupt is handled in world/service/player which are responsible for
# shutting down the eye process properly
signal.signal(signal.SIGINT, interrupt_handler)
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id + 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (600, 90 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
icon_bar_width = 50
window_size = None
camera_render_size = None
hdpi_factor = 1.0
# g_pool holds variables for this process
g_pool = SimpleNamespace()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = "capture"
g_pool.eye_id = eye_id
g_pool.process = f"eye{eye_id}"
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
default_detector_cls, available_detectors = available_detector_plugins(
)
plugins = (manager_classes + source_classes + available_detectors +
[PupilDetectorManager])
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
preferred_names = [
f"Pupil Cam3 ID{eye_id}",
f"Pupil Cam2 ID{eye_id}",
f"Pupil Cam1 ID{eye_id}",
]
if eye_id == 0:
preferred_names += ["HD-6000"]
default_capture_settings = (
"UVC_Source",
{
"preferred_names": preferred_names,
"frame_size": (320, 240),
"frame_rate": 120,
},
)
default_plugins = [
# TODO: extend with plugins
default_capture_settings,
("UVC_Manager", {}),
# Detector needs to be loaded first to set `g_pool.pupil_detector`
(default_detector_cls.__name__, {}),
("PupilDetectorManager", {}),
]
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x *= hdpi_factor
y *= hdpi_factor
g_pool.gui.update_mouse(x, y)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, g_pool.capture.frame_size)
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
for plugin in g_pool.plugins:
if plugin.on_drop(paths):
break
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir,
"user_settings_eye{}".format(eye_id)))
if VersionFormat(session_settings.get("version",
"0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.flip = session_settings.get("flip", False)
g_pool.display_mode = session_settings.get("display_mode",
"camera_image")
g_pool.display_mode_info_text = {
"camera_image":
"Raw eye camera image. This uses the least amount of CPU power",
"roi":
"Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
"algorithm":
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
}
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# Initialize glfw
glfw.glfwInit()
if hide_ui:
glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window
title = "Pupil Capture - eye {}".format(eye_id)
width, height = session_settings.get("window_size",
(640 + icon_bar_width, 480))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get("window_position",
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(
np.ones((1, 1), dtype=np.uint8) + 125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos="left")
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos="hidden")
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2],
label="Interface Size",
))
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width *= 2
f_height *= 2
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
def uroi_on_mouse_button(button, action, mods):
if g_pool.display_mode == "roi":
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
x, y = glfw.glfwGetCursorPos(main_window)
# pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
x *= hdpi_factor
y *= hdpi_factor
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(
pos,
g_pool.capture.frame_size) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,
g_pool.u_r.handle_size,
g_pool.u_r.handle_size):
# if the roi interacts we dont want
# the gui to interact as well
return
general_settings.append(ui.Button("Reset window size",
set_window_size))
general_settings.append(
ui.Switch("flip", g_pool, label="Flip image display"))
general_settings.append(
ui.Selector(
"display_mode",
g_pool,
setter=set_display_mode_info,
selection=["camera_image", "roi", "algorithm"],
labels=["Camera Image", "ROI", "Algorithm"],
label="Mode",
))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
toggle_general_settings(False)
plugins_to_load = session_settings.get("loaded_plugins",
default_plugins)
if overwrite_cap_settings:
# Ensure that overwrite_cap_settings takes preference over source plugins
# with incorrect settings that were loaded from session settings.
plugins_to_load.append(overwrite_cap_settings)
g_pool.plugins = Plugin_List(g_pool, plugins_to_load)
g_pool.writer = None
g_pool.u_r = UIRoi(
(g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get("roi")
if roi_user_settings and tuple(
roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 50)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 50)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = "jpeg"
frame_publish_format_recent_warning = False
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning("Process started.")
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification["subject"]
if subject.startswith("eye_process.should_stop"):
if notification["eye_id"] == eye_id:
break
elif subject == "recording.started":
if notification["record_eye"] and g_pool.capture.online:
record_path = notification["rec_path"]
raw_mode = notification["compression"]
start_time_synced = notification["start_time_synced"]
logger.info(
"Will save eye video to: {}".format(record_path))
video_path = os.path.join(record_path,
"eye{}.mp4".format(eye_id))
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(
video_path, start_time_synced)
elif hasattr(g_pool.capture._recent_frame,
"h264_buffer"):
g_pool.writer = H264Writer(
video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate,
)
else:
g_pool.writer = MPEG_Writer(
video_path, start_time_synced)
elif subject == "recording.stopped":
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
elif subject.startswith("meta.should_doc"):
ipc_socket.notify({
"subject": "meta.doc",
"actor": "eye{}".format(eye_id),
"doc": eye.__doc__,
})
elif subject.startswith("frame_publishing.started"):
should_publish_frames = True
frame_publish_format = notification.get("format", "jpeg")
elif subject.startswith("frame_publishing.stopped"):
should_publish_frames = False
frame_publish_format = "jpeg"
elif (subject.startswith("start_eye_plugin")
and notification["target"] == g_pool.process):
try:
g_pool.plugins.add(
g_pool.plugin_by_name[notification["name"]],
notification.get("args", {}),
)
except KeyError as err:
logger.error(f"Attempt to load unknown plugin: {err}")
for plugin in g_pool.plugins:
plugin.on_notify(notification)
event = {}
for plugin in g_pool.plugins:
plugin.recent_events(event)
frame = event.get("frame")
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
f_height,
f_width,
):
g_pool.pupil_detector.on_resolution_change(
(g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
g_pool.capture.frame_size,
)
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
assert data is not None
except (AttributeError, AssertionError, NameError):
if not frame_publish_format_recent_warning:
frame_publish_format_recent_warning = True
logger.warning(
'{}s are not compatible with format "{}"'.
format(type(frame), frame_publish_format))
else:
frame_publish_format_recent_warning = False
pupil_socket.send({
"topic":
"frame.eye.{}".format(eye_id),
"width":
frame.width,
"height":
frame.height,
"index":
frame.index,
"timestamp":
frame.timestamp,
"format":
frame_publish_format,
"__raw_data__": [data],
})
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1.0 / dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
result = event.get("pupil_detection_result", None)
if result is not None:
pupil_socket.send(result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
glViewport(0, 0, *camera_render_size)
for p in g_pool.plugins:
p.gl_display()
glViewport(0, 0, *camera_render_size)
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == "roi":
g_pool.u_r.draw_points(g_pool.gui.scale)
glViewport(0, 0, *window_size)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
try:
clipboard = glfw.glfwGetClipboardString(
main_window).decode()
except AttributeError: # clipboard is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(
main_window, user_input.clipboard.encode())
for button, action, mods in user_input.buttons:
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer.release()
g_pool.writer = None
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
# save session persistent settings
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["roi"] = g_pool.u_r.get()
session_settings["flip"] = g_pool.flip
session_settings["display_mode"] = g_pool.display_mode
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["version"] = str(g_pool.version)
if not hide_ui:
glfw.glfwRestoreWindow(
main_window) # need to do this for windows os
session_settings["window_position"] = glfw.glfwGetWindowPos(
main_window)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
for plugin in g_pool.plugins:
plugin.alive = False
g_pool.plugins.clean()
glfw.glfwDestroyWindow(main_window)
g_pool.gui.terminate()
glfw.glfwTerminate()
logger.info("Process shutting down.")
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url,ipc_push_url, user_dir, version, eye_id,overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=("notify",))
with Is_Alive_Manager(is_alive_flag,ipc_socket,eye_id):
#logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based,is_window_visible
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from uvc import get_time_monotonic, StreamError
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from av_writer import JPEG_Writer,AV_Writer
from video_capture import InitialisationError,StreamError, Fake_Source,EndofVideoFileError, source_classes, manager_classes
source_by_name = {src.class_name():src for src in source_classes}
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic()-g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window,w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager',{}))
if eye_id == 0:
cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
else:
cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]
# Initialize capture
default_settings = {
'source_class_name': 'UVC_Source',
'preferred_names' : cap_src,
'frame_size': (640,480),
'frame_rate': 90
}
settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
try:
cap = source_by_name[settings['source_class_name']](g_pool, **settings)
except (KeyError,InitialisationError) as e:
if isinstance(e,KeyError):
logger.warning('Incompatible capture setting encountered. Falling back to fake source.')
cap = Fake_Source(g_pool, **settings)
g_pool.iconified = False
g_pool.capture = cap
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip',False)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
writer = None
pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s"%eye_id
width,height = session_settings.get('window_size',g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width,height, title, None, None)
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
glfw.glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
general_settings.append(detector_selector)
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture.init_gui()
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__:c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
g_pool.capture_manager.init_gui()
def open_manager(manager_class):
g_pool.capture_manager.cleanup()
g_pool.capture_manager = manager_class(g_pool)
g_pool.capture_manager.init_gui()
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(0,ui.Selector(
'capture_manager',g_pool,
setter = open_manager,
getter = lambda: g_pool.capture_manager.__class__,
selection = manager_classes,
labels = [b.gui_name for b in manager_classes],
label = 'Manager'
))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
should_publish_frames = False
frame_publish_format = 'jpeg'
#create a timer to control window update frequency
window_update_timer = timer(1/60.)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t,notification = notify_sub.recv()
subject = notification['subject']
if subject == 'eye_process.should_stop':
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector,Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector,Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye']:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: %s"%record_path)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = JPEG_Writer(video_path,g_pool.capture.frame_rate)
else:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = AV_Writer(video_path,g_pool.capture.frame_rate)
timestamps = []
elif subject == 'recording.stopped':
if writer:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject':'meta.doc',
'actor':'eye%i'%eye_id,
'doc':eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format','jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
else:
g_pool.capture_manager.on_notify(notification)
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except StreamError as e:
logger.error("Error getting frame. Stopping eye process.")
logger.debug("Caught error: %s"%e)
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
g_pool.capture.seek_to_frame(0)
frame = g_pool.capture.get_frame()
g_pool.u_r = UIRoi((frame.height,frame.width))
g_pool.capture_manager.update(frame, {})
if should_publish_frames and frame.jpeg_buffer:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
pupil_socket.send('frame.eye.%s'%eye_id,{
'width': frame.width,
'height': frame.width,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s'%eye_id,result)
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
g_pool.capture.gl_display()
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
(int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
int(eye_ball['angle']),0,360,8)
except ValueError as e:
pass
else:
draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )
if result['confidence'] >0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
int(result['ellipse']['angle']),0,360,15)
confidence = result['confidence'] * 0.7 #scale it a little
draw_polyline(pts,1,RGBA(1.,0,0,confidence))
draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_gui()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
logger.info("Process shutting down.")
# Event loop
while glfwGetWindowParam(GLFW_OPENED): # and not quit.value
update_fps()
# Get an image from the grabber
# s, img = cap.read()
img = np.zeros((480,640,3))
# for p in g.plugins:
# p.update(img)
# g.plugins = [p for p in g.plugins if p.alive]
# render the screen
clear_gl_screen()
draw_gl_texture(img)
# render visual feedback from loaded plugins
# for p in g.plugins:
# p.gl_display()
atb.draw()
glfwSwapBuffers()
# end while running and clean-up
print "Browser closed"
glfwCloseWindow()
glfwTerminate()
def show_no_rec_window():
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
def on_drop(window,count,paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'"%new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window,True)
else:
logger.error("'%s' is not a valid pupil recording"%new_rec_dir)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file):
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
w,h = session_settings.get('window_size',(1280,720))
window_pos = session_settings.get('window_position',(0,0))
glfwWindowHint(GLFW_RESIZABLE,0)
window = glfwCreateWindow(w, h,'Pupil Player')
glfwWindowHint(GLFW_RESIZABLE,1)
glfwMakeContextCurrent(window)
glfwSetWindowPos(window,window_pos[0],window_pos[1])
glfwSetDropCallback(window,on_drop)
adjust_gl_view(w,h)
glfont = fontstash.Context()
glfont.add_font('roboto',get_roboto_font_path())
glfont.set_align_string(v_align="center",h_align="middle")
glfont.set_color_float((0.2,0.2,0.2,0.9))
basic_gl_setup()
glClearColor(0.5,.5,0.5,0.0)
text = 'Drop a recording directory onto this window.'
tip = '(Tip: You can drop a recording directory onto the app icon.)'
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
while not glfwWindowShouldClose(window):
clear_gl_screen()
glfont.set_blur(10.5)
glfont.set_color_float((0.0,0.0,0.0,1.))
glfont.set_size(w/25.)
glfont.draw_text(w/2,.3*h,text)
glfont.set_size(w/30.)
glfont.draw_text(w/2,.4*h,tip)
glfont.set_blur(0.96)
glfont.set_color_float((1.,1.,1.,1.))
glfont.set_size(w/25.)
glfont.draw_text(w/2,.3*h,text)
glfont.set_size(w/30.)
glfont.draw_text(w/2,.4*h,tip)
glfwSwapBuffers(window)
glfwPollEvents()
session_settings['window_position'] = glfwGetWindowPos(window)
session_settings['version'] = get_version(version_file)
session_settings.close()
del glfont
glfwDestroyWindow(window)
def eye(g_pool):
"""
this process needs a docstring
"""
# # callback functions
def on_resize(w, h):
atb.TwWindowSize(w, h)
adjust_gl_view(w, h)
def on_key(key, pressed):
if not atb.TwEventKeyboardGLFW(key, pressed):
if pressed:
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if not atb.TwEventCharGLFW(char, pressed):
pass
def on_button(button, pressed):
if not atb.TwEventMouseButtonGLFW(button, pressed):
if bar.draw_roi.value:
if pressed:
pos = glfwGetMousePos()
pos = normalize(pos, glfwGetWindowSize())
pos = denormalize(pos, (img.shape[1], img.shape[0])) # pos in img pixels
r.setStart(pos)
bar.draw_roi.value = 1
else:
bar.draw_roi.value = 0
def on_pos(x, y):
if atb.TwMouseMotion(x, y):
pass
if bar.draw_roi.value == 1:
pos = glfwGetMousePos()
pos = normalize(pos, glfwGetWindowSize())
pos = denormalize(pos, (img.shape[1], img.shape[0])) # pos in img pixels
r.setEnd(pos)
def on_scroll(pos):
if not atb.TwMouseWheel(pos):
pass
def on_close():
g_pool.quit.value = True
print "EYE Process closing from window"
# initialize capture, check if it works
cap = autoCreateCapture(g_pool.eye_src, g_pool.eye_size)
if cap is None:
print "EYE: Error could not create Capture"
return
s, img = cap.read_RGB()
if not s:
print "EYE: Error could not get image"
return
height, width = img.shape[:2]
# pupil object
pupil = Temp()
pupil.norm_coords = (0.0, 0.0)
pupil.image_coords = (0.0, 0.0)
pupil.ellipse = None
pupil.gaze_coords = (0.0, 0.0)
try:
pupil.pt_cloud = np.load("cal_pt_cloud.npy")
map_pupil = get_map_from_cloud(pupil.pt_cloud, g_pool.world_size) ###world video size here
except:
pupil.pt_cloud = None
def map_pupil(vector):
return vector
r = Roi(img.shape)
p_r = Roi(img.shape)
# local object
l_pool = Temp()
l_pool.calib_running = False
l_pool.record_running = False
l_pool.record_positions = []
l_pool.record_path = None
l_pool.writer = None
l_pool.region_r = 20
atb.init()
bar = Bar(
"Eye",
g_pool,
dict(
label="Controls",
help="eye detection controls",
color=(50, 50, 50),
alpha=100,
text="light",
position=(10, 10),
refresh=0.1,
size=(200, 300),
),
)
# add 4vl2 camera controls to a seperate ATB bar
if cap.controls is not None:
c_bar = atb.Bar(
name="Camera_Controls",
label=cap.name,
help="UVC Camera Controls",
color=(50, 50, 50),
alpha=100,
text="light",
position=(220, 10),
refresh=2.0,
size=(200, 200),
)
# c_bar.add_var("auto_refresher",vtype=atb.TW_TYPE_BOOL8,getter=cap.uvc_refresh_all,setter=None,readonly=True)
# c_bar.define(definition='visible=0', varname="auto_refresher")
sorted_controls = [c for c in cap.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type == "bool":
c_bar.add_var(name, vtype=atb.TW_TYPE_BOOL8, getter=control.get_val, setter=control.set_val)
elif control.type == "int":
c_bar.add_var(name, vtype=atb.TW_TYPE_INT32, getter=control.get_val, setter=control.set_val)
c_bar.define(definition="min=" + str(control.min), varname=name)
c_bar.define(definition="max=" + str(control.max), varname=name)
c_bar.define(definition="step=" + str(control.step), varname=name)
elif control.type == "menu":
if control.menu is None:
vtype = None
else:
vtype = atb.enum(name, control.menu)
c_bar.add_var(name, vtype=vtype, getter=control.get_val, setter=control.set_val)
if control.menu is None:
c_bar.define(definition="min=" + str(control.min), varname=name)
c_bar.define(definition="max=" + str(control.max), varname=name)
c_bar.define(definition="step=" + str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
# c_bar.define(definition='readonly=1',varname=control.name)
c_bar.add_button("refresh", cap.update_from_device)
c_bar.add_button("load defaults", cap.load_defaults)
else:
c_bar = None
# Initialize glfw
glfwInit()
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("Eye")
glfwSetWindowPos(800, 0)
if isinstance(g_pool.eye_src, str):
glfwSwapInterval(0) # turn of v-sync when using video as src for benchmarking
# register callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
glfwSetMouseButtonCallback(on_button)
glfwSetMousePosCallback(on_pos)
glfwSetMouseWheelCallback(on_scroll)
# gl_state settings
import OpenGL.GL as gl
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glPointSize(20)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
del gl
# event loop
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
bar.update_fps()
s, img = cap.read_RGB()
sleep(bar.sleep.value) # for debugging only
###IMAGE PROCESSING
gray_img = grayscale(img[r.lY : r.uY, r.lX : r.uX])
integral = cv2.integral(gray_img)
integral = np.array(integral, dtype=c_float)
x, y, w = eye_filter(integral)
if w > 0:
p_r.set((y, x, y + w, x + w))
else:
p_r.set((0, 0, -1, -1))
# create view into the gray_img with the bounds of the rough pupil estimation
pupil_img = gray_img[p_r.lY : p_r.uY, p_r.lX : p_r.uX]
# pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT,(5,5)),iterations=2)
if True:
hist = cv2.calcHist(
[pupil_img], [0], None, [256], [0, 256]
) # (images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:, 0] > 40] # every color seen in more than 40 pixels
if spikes.shape[0] > 0:
lowest_spike = spikes.min()
offset = 40
##display the histogram
sx, sy = 100, 1
colors = ((255, 0, 0), (0, 0, 255), (0, 255, 255))
h, w, chan = img.shape
# normalize
hist *= 1.0 / hist.max()
for i, h in zip(bins, hist[:, 0]):
c = colors[1]
cv2.line(img, (w, int(i * sy)), (w - int(h * sx), int(i * sy)), c)
cv2.line(img, (w, int(lowest_spike * sy)), (int(w - 0.5 * sx), int(lowest_spike * sy)), colors[0])
cv2.line(
img,
(w, int((lowest_spike + offset) * sy)),
(int(w - 0.5 * sx), int((lowest_spike + offset) * sy)),
colors[2],
)
# # k-means on the histogram finds peaks but thats no good for us...
# term_crit = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
# compactness, bestLabels, centers = cv2.kmeans(data=hist, K=2, criteria=term_crit, attempts=10, flags=cv2.KMEANS_RANDOM_CENTERS)
# cv2.line(img,(0,1),(int(compactness),1),(0,0,0))
# good_cluster = np.argmax(centers)
# # A = hist[bestLabels.ravel() == good_cluster]
# # B = hist[bestLabels.ravel() != good_cluster]
# bins = np.arange(hist.shape[0])
# good_bins = bins[bestLabels.ravel() == good_cluster]
# good_bins_mean = good_bins.sum()/good_bins.shape[0]
# good_bins_min = good_bins.min()
# h,w,chan = img.shape
# for h, i, label in zip(hist[:,0],range(hist.shape[0]), bestLabels.ravel()):
# c = colors[label]
# cv2.line(img,(w,int(i*sy)),(w-int(h*sx),int(i*sy)),c)
else:
# direct k-means on the image is best but expensive
Z = pupil_img[:: w / 30 + 1, :: w / 30 + 1].reshape((-1, 1))
Z = np.float32(Z)
# define criteria, number of clusters(K) and apply kmeans()
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 20, 2.0)
K = 5
ret, label, center = cv2.kmeans(Z, K, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)
offset = 0
center.sort(axis=0)
lowest_spike = int(center[1])
# # Now convert back into uint8, and make original image
# center = np.uint8(center)
# res = center[label.flatten()]
# binary_img = res.reshape((pupil_img.shape))
# binary_img = bin_thresholding(binary_img,image_upper=res.min()+1)
# bar.bin_thresh.value = res.min()+1
bar.bin_thresh.value = lowest_spike
binary_img = bin_thresholding(pupil_img, image_upper=lowest_spike + offset)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
cv2.dilate(binary_img, kernel, binary_img, iterations=2)
spec_mask = bin_thresholding(pupil_img, image_upper=250)
cv2.erode(spec_mask, kernel, spec_mask, iterations=1)
if bar.blur.value > 1:
pupil_img = cv2.medianBlur(pupil_img, bar.blur.value)
# create contours using Canny edge dectetion
contours = cv2.Canny(
pupil_img,
bar.canny_thresh.value,
bar.canny_thresh.value * bar.canny_ratio.value,
apertureSize=bar.canny_aperture.value,
)
# remove contours in areas not dark enough and where the glint (spectral refelction from IR leds)
contours = cv2.min(contours, spec_mask)
contours = cv2.min(contours, binary_img)
# Ellipse fitting from countours
result = fit_ellipse(
img[r.lY : r.uY, r.lX : r.uX][p_r.lY : p_r.uY, p_r.lX : p_r.uX],
contours,
binary_img,
target_size=bar.pupil_size.value,
size_tolerance=bar.pupil_size_tolerance.value,
)
# # Vizualizations
overlay = cv2.cvtColor(pupil_img, cv2.COLOR_GRAY2RGB) # create an RGB view onto the gray pupil ROI
overlay[:, :, 0] = cv2.max(pupil_img, contours) # green channel
overlay[:, :, 2] = cv2.max(pupil_img, binary_img) # blue channel
overlay[:, :, 1] = cv2.min(pupil_img, spec_mask) # red channel
# draw a blue dotted frame around the automatic pupil ROI in overlay...
overlay[::2, 0] = 0, 0, 255
overlay[::2, -1] = 0, 0, 255
overlay[0, ::2] = 0, 0, 255
overlay[-1, ::2] = 0, 0, 255
# and a solid (white) frame around the user defined ROI
gray_img[:, 0] = 255
gray_img[:, -1] = 255
gray_img[0, :] = 255
gray_img[-1, :] = 255
if bar.display.value == 0:
img = img
elif bar.display.value == 1:
img[r.lY : r.uY, r.lX : r.uX] = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
elif bar.display.value == 2:
img[r.lY : r.uY, r.lX : r.uX] = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
img[r.lY : r.uY, r.lX : r.uX][p_r.lY : p_r.uY, p_r.lX : p_r.uX] = overlay
elif bar.display.value == 3:
img = cv2.cvtColor(pupil_img, cv2.COLOR_GRAY2RGB)
else:
pass
if result is not None:
pupil.ellipse, others = result
pupil.image_coords = r.add_vector(p_r.add_vector(pupil.ellipse["center"]))
# update pupil size,angle and ratio for the ellipse filter algorithm
bar.pupil_size.value = bar.pupil_size.value + 0.5 * (pupil.ellipse["major"] - bar.pupil_size.value)
bar.pupil_ratio.value = bar.pupil_ratio.value + 0.7 * (pupil.ellipse["ratio"] - bar.pupil_ratio.value)
bar.pupil_angle.value = bar.pupil_angle.value + 1.0 * (pupil.ellipse["angle"] - bar.pupil_angle.value)
# if pupil found tighten the size tolerance
bar.pupil_size_tolerance.value -= 1
bar.pupil_size_tolerance.value = max(10, min(50, bar.pupil_size_tolerance.value))
# clamp pupil size
bar.pupil_size.value = max(20, min(300, bar.pupil_size.value))
# normalize
pupil.norm_coords = normalize(pupil.image_coords, (img.shape[1], img.shape[0]), flip_y=True)
# from pupil to gaze
pupil.gaze_coords = map_pupil(pupil.norm_coords)
g_pool.gaze_x.value, g_pool.gaze_y.value = pupil.gaze_coords
else:
pupil.ellipse = None
g_pool.gaze_x.value, g_pool.gaze_y.value = 0.0, 0.0
pupil.gaze_coords = None # whithout this line the last know pupil position is recorded if none is found
bar.pupil_size_tolerance.value += 1
###CALIBRATION###
# Initialize Calibration (setup variables and lists)
if g_pool.calibrate.value and not l_pool.calib_running:
l_pool.calib_running = True
pupil.pt_cloud = []
# While Calibrating...
if l_pool.calib_running and ((g_pool.ref_x.value != 0) or (g_pool.ref_y.value != 0)) and pupil.ellipse:
pupil.pt_cloud.append([pupil.norm_coords[0], pupil.norm_coords[1], g_pool.ref_x.value, g_pool.ref_y.value])
# Calculate mapping coefs
if not g_pool.calibrate.value and l_pool.calib_running:
l_pool.calib_running = 0
if pupil.pt_cloud: # some data was actually collected
print "Calibrating with", len(pupil.pt_cloud), "collected data points."
map_pupil = get_map_from_cloud(np.array(pupil.pt_cloud), g_pool.world_size, verbose=True)
np.save("cal_pt_cloud.npy", np.array(pupil.pt_cloud))
###RECORDING###
# Setup variables and lists for recording
if g_pool.pos_record.value and not l_pool.record_running:
l_pool.record_path = g_pool.eye_rx.recv()
print "l_pool.record_path: ", l_pool.record_path
video_path = path.join(l_pool.record_path, "eye.avi")
# FFV1 -- good speed lossless big file
# DIVX -- good speed good compression medium file
if bar.record_eye.value:
l_pool.writer = cv2.VideoWriter(
video_path, cv2.cv.CV_FOURCC(*"DIVX"), bar.fps.value, (img.shape[1], img.shape[0])
)
l_pool.record_positions = []
l_pool.record_running = True
# While recording...
if l_pool.record_running:
if pupil.gaze_coords is not None:
l_pool.record_positions.append(
[
pupil.gaze_coords[0],
pupil.gaze_coords[1],
pupil.norm_coords[0],
pupil.norm_coords[1],
bar.dt,
g_pool.frame_count_record.value,
]
)
if l_pool.writer is not None:
l_pool.writer.write(cv2.cvtColor(img, cv2.cv.COLOR_BGR2RGB))
# Done Recording: Save values and flip switch to off for recording
if not g_pool.pos_record.value and l_pool.record_running:
positions_path = path.join(l_pool.record_path, "gaze_positions.npy")
cal_pt_cloud_path = path.join(l_pool.record_path, "cal_pt_cloud.npy")
np.save(positions_path, np.asarray(l_pool.record_positions))
try:
np.save(cal_pt_cloud_path, np.asarray(pupil.pt_cloud))
except:
print "Warning: No calibration data associated with this recording."
l_pool.writer = None
l_pool.record_running = False
### GL-drawing
clear_gl_screen()
draw_gl_texture(img)
if bar.draw_pupil and pupil.ellipse:
pts = cv2.ellipse2Poly(
(int(pupil.image_coords[0]), int(pupil.image_coords[1])),
(int(pupil.ellipse["axes"][0] / 2), int(pupil.ellipse["axes"][1] / 2)),
int(pupil.ellipse["angle"]),
0,
360,
15,
)
draw_gl_polyline(pts, (1.0, 0, 0, 0.5))
draw_gl_point_norm(pupil.norm_coords, (1.0, 0.0, 0.0, 0.5))
atb.draw()
glfwSwapBuffers()
# end while running
print "EYE Process closed"
r.save()
bar.save()
atb.terminate()
glfwCloseWindow()
glfwTerminate()
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version, debug):
# general imports
import logging
# networking
import zmq
import zmq_tools
from time import sleep
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
import glfw
import gl_utils
from OpenGL.GL import glClearColor
from version_utils import VersionFormat
from file_methods import Persistent_Dict
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
import player_methods as pm
from pupil_recording import (
assert_valid_recording_type,
InvalidRecordingException,
)
from pupil_recording.update import update_recording
process_was_interrupted = False
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
nonlocal process_was_interrupted
process_was_interrupted = True
signal.signal(signal.SIGINT, interrupt_handler)
def on_drop(window, count, paths):
nonlocal rec_dir
rec_dir = paths[0].decode("utf-8")
if rec_dir:
try:
assert_valid_recording_type(rec_dir)
except InvalidRecordingException as err:
logger.error(str(err))
rec_dir = None
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version",
"0.0")) != app_version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
w, h = session_settings.get("window_size", (1280, 720))
window_pos = session_settings.get("window_position",
window_position_default)
glfw.glfwInit()
glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0)
window = glfw.glfwCreateWindow(w, h, "Pupil Player")
glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1)
glfw.glfwMakeContextCurrent(window)
glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1])
glfw.glfwSetDropCallback(window, on_drop)
glfont = fontstash.Context()
glfont.add_font("roboto", get_roboto_font_path())
glfont.set_align_string(v_align="center", h_align="middle")
glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
gl_utils.basic_gl_setup()
glClearColor(0.5, 0.5, 0.5, 0.0)
text = "Drop a recording directory onto this window."
tip = "(Tip: You can drop a recording directory onto the app icon.)"
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
def display_string(string, font_size, center_y):
x = w / 2 * hdpi_factor
y = center_y * hdpi_factor
glfont.set_size(font_size * hdpi_factor)
glfont.set_blur(10.5)
glfont.set_color_float((0.0, 0.0, 0.0, 1.0))
glfont.draw_text(x, y, string)
glfont.set_blur(0.96)
glfont.set_color_float((1.0, 1.0, 1.0, 1.0))
glfont.draw_text(x, y, string)
while not glfw.glfwWindowShouldClose(
window) and not process_was_interrupted:
fb_size = glfw.glfwGetFramebufferSize(window)
hdpi_factor = glfw.getHDPIFactor(window)
gl_utils.adjust_gl_view(*fb_size)
if rec_dir:
try:
assert_valid_recording_type(rec_dir)
logger.info(
"Starting new session with '{}'".format(rec_dir))
text = "Updating recording format."
tip = "This may take a while!"
except InvalidRecordingException as err:
logger.error(str(err))
if err.recovery:
text = err.reason
tip = err.recovery
else:
text = "Invalid recording"
tip = err.reason
rec_dir = None
gl_utils.clear_gl_screen()
display_string(text, font_size=51, center_y=216)
for idx, line in enumerate(tip.split("\n")):
tip_font_size = 42
center_y = 288 + tip_font_size * idx * 1.2
display_string(line,
font_size=tip_font_size,
center_y=center_y)
glfw.glfwSwapBuffers(window)
if rec_dir:
try:
update_recording(rec_dir)
except AssertionError as err:
logger.error(str(err))
tip = "Oops! There was an error updating the recording."
rec_dir = None
except InvalidRecordingException as err:
logger.error(str(err))
if err.recovery:
text = err.reason
tip = err.recovery
else:
text = "Invalid recording"
tip = err.reason
rec_dir = None
else:
glfw.glfwSetWindowShouldClose(window, True)
glfw.glfwPollEvents()
session_settings["window_position"] = glfw.glfwGetWindowPos(window)
session_settings.close()
glfw.glfwDestroyWindow(window)
if rec_dir:
ipc_pub.notify({
"subject": "player_process.should_start",
"rec_dir": rec_dir
})
except Exception:
import traceback
trace = traceback.format_exc()
logger.error(
"Process player_drop crashed with trace:\n{}".format(trace))
finally:
sleep(1.0)
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version):
# general imports
import logging
# networking
import zmq
import zmq_tools
from time import sleep
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
import glfw
import gl_utils
from OpenGL.GL import glClearColor
from version_utils import VersionFormat
from file_methods import Persistent_Dict
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
from player_methods import is_pupil_rec_dir, update_recording_to_recent
def on_drop(window, count, paths):
nonlocal rec_dir
rec_dir = paths[0].decode('utf-8')
if rec_dir:
if not is_pupil_rec_dir(rec_dir):
rec_dir = None
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version",
'0.0')) != app_version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
w, h = session_settings.get('window_size', (1280, 720))
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwInit()
glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0)
window = glfw.glfwCreateWindow(w, h, 'Pupil Player')
glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1)
glfw.glfwMakeContextCurrent(window)
glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1])
glfw.glfwSetDropCallback(window, on_drop)
glfont = fontstash.Context()
glfont.add_font('roboto', get_roboto_font_path())
glfont.set_align_string(v_align="center", h_align="middle")
glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
gl_utils.basic_gl_setup()
glClearColor(0.5, .5, 0.5, 0.0)
text = 'Drop a recording directory onto this window.'
tip = '(Tip: You can drop a recording directory onto the app icon.)'
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
while not glfw.glfwWindowShouldClose(window):
fb_size = glfw.glfwGetFramebufferSize(window)
hdpi_factor = float(fb_size[0] / glfw.glfwGetWindowSize(window)[0])
gl_utils.adjust_gl_view(*fb_size)
if rec_dir:
if is_pupil_rec_dir(rec_dir):
logger.info(
"Starting new session with '{}'".format(rec_dir))
text = "Updating recording format."
tip = "This may take a while!"
else:
logger.error(
"'{}' is not a valid pupil recording".format(rec_dir))
tip = "Oops! That was not a valid recording."
rec_dir = None
gl_utils.clear_gl_screen()
glfont.set_blur(10.5)
glfont.set_color_float((0.0, 0.0, 0.0, 1.))
glfont.set_size(w / 25. * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, .3 * h * hdpi_factor, text)
glfont.set_size(w / 30. * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, .4 * h * hdpi_factor, tip)
glfont.set_blur(0.96)
glfont.set_color_float((1., 1., 1., 1.))
glfont.set_size(w / 25. * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, .3 * h * hdpi_factor, text)
glfont.set_size(w / 30. * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, .4 * h * hdpi_factor, tip)
glfw.glfwSwapBuffers(window)
if rec_dir:
update_recording_to_recent(rec_dir)
glfw.glfwSetWindowShouldClose(window, True)
glfw.glfwPollEvents()
session_settings['window_position'] = glfw.glfwGetWindowPos(window)
session_settings.close()
glfw.glfwDestroyWindow(window)
if rec_dir:
ipc_pub.notify({
"subject": "player_process.should_start",
"rec_dir": rec_dir
})
except:
import traceback
trace = traceback.format_exc()
logger.error(
'Process player_drop crashed with trace:\n{}'.format(trace))
finally:
sleep(1.0)
def world(g_pool):
"""world
"""
# Callback functions
def on_resize(w, h):
atb.TwWindowSize(w, h);
adjust_gl_view(w,h)
def on_key(key, pressed):
if not atb.TwEventKeyboardGLFW(key,pressed):
if pressed:
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if not atb.TwEventCharGLFW(char,pressed):
pass
def on_button(button, pressed):
if not atb.TwEventMouseButtonGLFW(button,pressed):
if pressed:
pos = glfwGetMousePos()
pos = normalize(pos,glfwGetWindowSize())
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g.plugins:
p.on_click(pos)
def on_pos(x, y):
if atb.TwMouseMotion(x,y):
pass
def on_scroll(pos):
if not atb.TwMouseWheel(pos):
pass
def on_close():
g_pool.quit.value = True
print "WORLD Process closing from window"
# load session persistent settings
session_settings = shelve.open('user_settings_world',protocol=2)
def load(var_name,default):
try:
return session_settings[var_name]
except:
return default
def save(var_name,var):
session_settings[var_name] = var
# gaze object
gaze = Temp()
gaze.map_coords = (0., 0.)
gaze.image_coords = (0., 0.)
# Initialize capture, check if it works
cap = autoCreateCapture(g_pool.world_src, g_pool.world_size,24)
if cap is None:
print "WORLD: Error could not create Capture"
return
frame = cap.get_frame()
if frame.img is None:
print "WORLD: Error could not get image"
return
height,width = frame.img.shape[:2]
# helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1 / dt - bar.fps.value)
def set_window_size(mode,data):
height,width = frame.img.shape[:2]
ratio = (1,.75,.5,.25)[mode]
w,h = int(width*ratio),int(height*ratio)
glfwSetWindowSize(w,h)
data.value=mode # update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def open_calibration(selection,data):
# prepare destruction of old ref_detector.
if g.current_ref_detector:
g.current_ref_detector.alive = False
# remove old ref detector from list of plugins
g.plugins = [p for p in g.plugins if p.alive]
print "selected: ",reference_detectors.name_by_index[selection]
g.current_ref_detector = reference_detectors.detector_by_index[selection](global_calibrate=g_pool.calibrate,
shared_pos=g_pool.ref,
screen_marker_pos = g_pool.marker,
screen_marker_state = g_pool.marker_state,
atb_pos=bar.next_atb_pos)
g.plugins.append(g.current_ref_detector)
# save the value for atb bar
data.value=selection
def toggle_record_video():
if any([True for p in g.plugins if isinstance(p,recorder.Recorder)]):
for p in g.plugins:
if isinstance(p,recorder.Recorder):
p.alive = False
else:
# set up folder within recordings named by user input in atb
if not bar.rec_name.value:
bar.rec_name.value = recorder.get_auto_name()
recorder_instance = recorder.Recorder(bar.rec_name.value, bar.fps.value, frame.img.shape, g_pool.pos_record, g_pool.eye_tx)
g.plugins.append(recorder_instance)
def toggle_show_calib_result():
if any([True for p in g.plugins if isinstance(p,Show_Calibration)]):
for p in g.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
else:
calib = Show_Calibration(frame.img.shape)
g.plugins.append(calib)
def show_calib_result():
# kill old if any
if any([True for p in g.plugins if isinstance(p,Show_Calibration)]):
for p in g.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
g.plugins = [p for p in g.plugins if p.alive]
# make new
calib = Show_Calibration(frame.img.shape)
g.plugins.append(calib)
def hide_calib_result():
if any([True for p in g.plugins if isinstance(p,Show_Calibration)]):
for p in g.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
# Initialize ant tweak bar - inherits from atb.Bar
atb.init()
bar = atb.Bar(name = "World", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
text='light', position=(10, 10),refresh=.3, size=(300, 200))
bar.next_atb_pos = (10,220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.calibration_type = c_int(load("calibration_type",0))
bar.show_calib_result = c_bool(0)
bar.record_video = c_bool(0)
bar.record_running = c_bool(0)
bar.play = g_pool.play
bar.window_size = c_int(load("window_size",0))
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
bar.calibrate_type_enum = atb.enum("Calibration Method",reference_detectors.index_by_name)
bar.rec_name = create_string_buffer(512)
bar.rec_name.value = recorder.get_auto_name()
# play and record can be tied together via pointers to the objects
# bar.play = bar.record_video
bar.add_var("fps", bar.fps, step=1., readonly=True)
bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=bar.calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.")
bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.")
bar.add_var("session name",bar.rec_name, group="Recording", help="creates folder Data_Name_XXX, where xxx is an increasing number")
bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording")
bar.add_separator("Sep1")
bar.add_var("play video", bar.play, help="play a video in the Player window")
bar.add_var("exit", g_pool.quit)
# add uvc camera controls to a seperate ATB bar
cap.create_atb_bar(pos=(320,10))
# create container for globally scoped vars (within world)
g = Temp()
g.plugins = []
g.current_ref_detector = None
open_calibration(bar.calibration_type.value,bar.calibration_type)
# Initialize glfw
glfwInit()
height,width = frame.img.shape[:2]
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("World")
glfwSetWindowPos(0,0)
#set the last saved window size
set_window_size(bar.window_size.value,bar.window_size)
# Register callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
glfwSetMouseButtonCallback(on_button)
glfwSetMousePosCallback(on_pos)
glfwSetMouseWheelCallback(on_scroll)
# gl_state settings
import OpenGL.GL as gl
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
del gl
# Event loop
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
# Get input characters entered in player
if g_pool.player_input.value:
player_input = g_pool.player_input.value
g_pool.player_input.value = 0
on_char(player_input,True)
# Get an image from the grabber
frame = cap.get_frame()
update_fps()
for p in g.plugins:
p.update(frame)
g.plugins = [p for p in g.plugins if p.alive]
g_pool.player_refresh.set()
# render the screen
clear_gl_screen()
draw_gl_texture(frame.img)
# render visual feedback from loaded plugins
for p in g.plugins:
p.gl_display()
# update gaze point from shared variable pool and draw on screen. If both coords are 0: no pupil pos was detected.
if not g_pool.gaze[:] == [0.,0.]:
draw_gl_point_norm(g_pool.gaze[:],color=(1.,0.,0.,0.5))
atb.draw()
glfwSwapBuffers()
# end while running and clean-up
# de-init all running plugins
for p in g.plugins:
p.alive = False
g.plugins = [p for p in g.plugins if p.alive]
save('window_size',bar.window_size.value)
save('calibration_type',bar.calibration_type.value)
session_settings.close()
cap.close()
glfwCloseWindow()
glfwTerminate()
print "WORLD Process closed"
def eye(g_pool,cap_src,cap_size):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye.log'),mode='w')
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.WARNING)
# create formatter and add it to the handlers
formatter = logging.Formatter('EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('E Y E Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# Callback functions
def on_resize(window,w, h):
adjust_gl_view(w,h,window)
norm_size = normalize((w,h),glfwGetWindowSize(window))
fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
atb.TwWindowSize(*map(int,fb_size))
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key,int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
def on_char(window,char):
if not atb.TwEventCharGLFW(char,1):
pass
def on_button(window,button, action, mods):
if not atb.TwEventMouseButtonGLFW(button,int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels
u_r.setStart(pos)
bar.draw_roi.value = 1
else:
bar.draw_roi.value = 0
def on_pos(window,x, y):
norm_pos = normalize((x,y),glfwGetWindowSize(window))
fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window))
if atb.TwMouseMotion(int(fb_x),int(fb_y)):
pass
if bar.draw_roi.value == 1:
pos = denormalize(norm_pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels
u_r.setEnd(pos)
def on_scroll(window,x,y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# Helper functions called by the main atb bar
def start_roi():
bar.display.value = 1
bar.draw_roi.value = 2
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1. / dt - bar.fps.value)
bar.dt.value = dt
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
# load session persistent settings
session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_eye'),protocol=2)
def load(var_name,default):
return session_settings.get(var_name,default)
def save(var_name,var):
session_settings[var_name] = var
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size,timebase=g_pool.timebase)
if cap is None:
logger.error("Did not receive valid Capture")
return
# check if it works
frame = cap.get_frame()
if frame.img is None:
logger.error("Could not retrieve image from capture")
cap.close()
return
height,width = frame.img.shape[:2]
u_r = Roi(frame.img.shape)
u_r.set(load('roi',default=None))
writer = None
pupil_detector = Canny_Detector(g_pool)
atb.init()
# Create main ATB Controls
bar = atb.Bar(name = "Eye", label="Display",
help="Scene controls", color=(50, 50, 50), alpha=100,
text='light', position=(10, 10),refresh=.3, size=(200, 100))
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.dt = c_float(0.0)
bar.sleep = c_float(0.0)
bar.display = c_int(load('bar.display',0))
bar.draw_pupil = c_bool(load('bar.draw_pupil',True))
bar.draw_roi = c_int(0)
dispay_mode_enum = atb.enum("Mode",{"Camera Image":0,
"Region of Interest":1,
"Algorithm":2,
"CPU Save": 3})
bar.add_var("FPS",bar.fps, step=1.,readonly=True)
bar.add_var("Mode", bar.display,vtype=dispay_mode_enum, help="select the view-mode")
bar.add_var("Show_Pupil_Point", bar.draw_pupil)
bar.add_button("Draw_ROI", start_roi, help="drag on screen to select a region of interest")
bar.add_var("SlowDown",bar.sleep, step=0.01,min=0.0)
bar.add_var("SaveSettings&Exit", g_pool.quit)
cap.create_atb_bar(pos=(220,10))
# create a bar for the detector
pupil_detector.create_atb_bar(pos=(10,120))
glfwInit()
window = glfwCreateWindow(width, height, "Eye", None, None)
glfwMakeContextCurrent(window)
# Register callbacks window
glfwSetWindowSizeCallback(window,on_resize)
glfwSetWindowCloseCallback(window,on_close)
glfwSetKeyCallback(window,on_key)
glfwSetCharCallback(window,on_char)
glfwSetMouseButtonCallback(window,on_button)
glfwSetCursorPosCallback(window,on_pos)
glfwSetScrollCallback(window,on_scroll)
glfwSetWindowPos(window,800,0)
on_resize(window,width,height)
# gl_state settings
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
# event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
update_fps()
sleep(bar.sleep.value) # for debugging only
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if g_pool.eye_rx.poll():
command = g_pool.eye_rx.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s"%record_path)
video_path = os.path.join(record_path, "eye.avi")
timestamps_path = os.path.join(record_path, "eye_timestamps.npy")
writer = cv2.VideoWriter(video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (frame.img.shape[1], frame.img.shape[0]))
timestamps = []
else:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write(frame.img)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(frame,user_roi=u_r,visualize=bar.display.value == 2)
# stream the result
g_pool.pupil_queue.put(result)
# VISUALIZATION direct visualizations on the frame.img data
if bar.display.value == 1:
# and a solid (white) frame around the user defined ROI
r_img = frame.img[u_r.lY:u_r.uY,u_r.lX:u_r.uX]
r_img[:,0] = 255,255,255
r_img[:,-1]= 255,255,255
r_img[0,:] = 255,255,255
r_img[-1,:]= 255,255,255
# GL-drawing
clear_gl_screen()
draw_gl_texture(frame.img,update=bar.display.value != 3)
if result['norm_pupil'] is not None and bar.draw_pupil.value:
if result.has_key('axes'):
pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
(int(result["axes"][0]/2),int(result["axes"][1]/2)),
int(result["angle"]),0,360,15)
draw_gl_polyline(pts,(1.,0,0,.5))
draw_gl_point_norm(result['norm_pupil'],color=(1.,0.,0.,0.5))
atb.draw()
glfwSwapBuffers(window)
glfwPollEvents()
# END while running
# in case eye reconding was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
# save session persistent settings
save('roi',u_r.get())
save('bar.display',bar.display.value)
save('bar.draw_pupil',bar.draw_pupil.value)
session_settings.close()
pupil_detector.cleanup()
cap.close()
atb.terminate()
glfwDestroyWindow(window)
glfwTerminate()
#flushing queue incase world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, version, eye_id, cap_src):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):
#logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, make_coord_system_eye_camera_based
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from av_writer import JPEG_Writer, AV_Writer
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D
}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.timebase = timebase
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos,
(frame.width, frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size + 40,
g_pool.u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version",
VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode',
'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."
}
g_pool.u_r = UIRoi((cap.frame_size[1], cap.frame_size[0]))
g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1], new_size[0]))
cap.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,
pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s" % eye_id
width, height = session_settings.get('window_size', cap.frame_size)
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
glfw.glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector(
'pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Detector_2D, Detector_3D],
labels=['C++ 2d detector', 'C++ 3d detector'],
label="Detection method")
general_settings.append(detector_selector)
# let detector add its GUI
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1 / 60.)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification['subject']
if subject == 'eye_process.should_stop':
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye']:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(
record_path, "eye%s_timestamps.npy" % eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path, cap.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = AV_Writer(video_path, cap.frame_rate)
timestamps = []
elif subject == 'recording.stopped':
if writer:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject': 'meta.doc',
'actor': 'eye%i' % eye_id,
'doc': eye.__doc__
})
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
cap.seek_to_frame(0)
frame = cap.get_frame()
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s' % eye_id, result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based(
(frame.height, frame.width, 3), g_pool.flip)
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly(
(int(eye_ball['center'][0]),
int(eye_ball['center'][1])),
(int(eye_ball['axes'][0] / 2),
int(eye_ball['axes'][1] / 2)),
int(eye_ball['angle']), 0, 360, 8)
except ValueError as e:
pass
else:
draw_polyline(
pts, 2,
RGBA(0., .9, .1, result['model_confidence']))
if result['confidence'] > 0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
confidence = result[
'confidence'] * 0.7 #scale it a little
draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., confidence),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize(
) #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = g_pool.version
session_settings[
'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings[
'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
logger.info("Process shutting down.")
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version, debug):
# general imports
import logging
# networking
import zmq
import zmq_tools
from time import sleep
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
import glfw
from gl_utils import GLFWErrorReporting
GLFWErrorReporting.set_default()
import gl_utils
from OpenGL.GL import glClearColor
from version_utils import parse_version
from file_methods import Persistent_Dict
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
import player_methods as pm
from pupil_recording import (
assert_valid_recording_type,
InvalidRecordingException,
)
from pupil_recording.update import update_recording
process_was_interrupted = False
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
nonlocal process_was_interrupted
process_was_interrupted = True
signal.signal(signal.SIGINT, interrupt_handler)
def on_drop(window, paths):
nonlocal rec_dir
rec_dir = paths[0]
if rec_dir:
try:
assert_valid_recording_type(rec_dir)
except InvalidRecordingException as err:
logger.error(str(err))
rec_dir = None
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if parse_version(session_settings.get("version",
"0.0")) != app_version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
w, h = session_settings.get("window_size", (1280, 720))
glfw.init()
glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
glfw.window_hint(glfw.RESIZABLE, 0)
window = glfw.create_window(w, h, "Pupil Player", None, None)
glfw.window_hint(glfw.RESIZABLE, 1)
glfw.make_context_current(window)
window_position_manager = gl_utils.WindowPositionManager()
window_pos = window_position_manager.new_window_position(
window=window,
default_position=window_position_default,
previous_position=session_settings.get("window_position", None),
)
glfw.set_window_pos(window, window_pos[0], window_pos[1])
glfw.set_drop_callback(window, on_drop)
glfont = fontstash.Context()
glfont.add_font("roboto", get_roboto_font_path())
glfont.set_align_string(v_align="center", h_align="middle")
glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
gl_utils.basic_gl_setup()
glClearColor(0.5, 0.5, 0.5, 0.0)
text = "Drop a recording directory onto this window."
tip = "(Tip: You can drop a recording directory onto the app icon.)"
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
def display_string(string, font_size, center_y):
x = w / 2 * content_scale
y = center_y * content_scale
glfont.set_size(font_size * content_scale)
glfont.set_blur(10.5)
glfont.set_color_float((0.0, 0.0, 0.0, 1.0))
glfont.draw_text(x, y, string)
glfont.set_blur(0.96)
glfont.set_color_float((1.0, 1.0, 1.0, 1.0))
glfont.draw_text(x, y, string)
def display_multiline_string(string, font_size, top_y, split_chr="\n"):
for idx, line in enumerate(string.split(split_chr)):
center_y = top_y + font_size * idx * 1.2
display_string(line, font_size=font_size, center_y=center_y)
bottom_y = top_y + font_size * (idx + 1) * 1.2
return bottom_y
while not glfw.window_should_close(
window) and not process_was_interrupted:
fb_size = glfw.get_framebuffer_size(window)
content_scale = gl_utils.get_content_scale(window)
gl_utils.adjust_gl_view(*fb_size)
if rec_dir:
try:
assert_valid_recording_type(rec_dir)
logger.info(
"Starting new session with '{}'".format(rec_dir))
text = "Updating recording format."
tip = "This may take a while!"
except InvalidRecordingException as err:
logger.error(str(err))
if err.recovery:
text = err.reason
tip = err.recovery
else:
text = "Invalid recording"
tip = err.reason
rec_dir = None
gl_utils.clear_gl_screen()
top_y = display_multiline_string(text, font_size=51, top_y=216)
display_multiline_string(tip, font_size=42, top_y=top_y + 50)
glfw.swap_buffers(window)
if rec_dir:
try:
update_recording(rec_dir)
except AssertionError as err:
logger.error(str(err))
tip = "Oops! There was an error updating the recording."
rec_dir = None
except InvalidRecordingException as err:
logger.error(str(err))
if err.recovery:
text = err.reason
tip = err.recovery
else:
text = "Invalid recording"
tip = err.reason
rec_dir = None
else:
glfw.set_window_should_close(window, True)
glfw.poll_events()
session_settings["window_position"] = glfw.get_window_pos(window)
session_settings.close()
glfw.destroy_window(window)
if rec_dir:
ipc_pub.notify({
"subject": "player_process.should_start",
"rec_dir": rec_dir
})
except Exception:
import traceback
trace = traceback.format_exc()
logger.error(
"Process player_drop crashed with trace:\n{}".format(trace))
finally:
sleep(1.0)
def player_drop(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version):
# general imports
import logging
# networking
import zmq
import zmq_tools
from time import sleep
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
import glfw
import gl_utils
from OpenGL.GL import glClearColor
from version_utils import VersionFormat
from file_methods import Persistent_Dict
from pyglui.pyfontstash import fontstash
from pyglui.ui import get_roboto_font_path
import player_methods as pm
import update_methods as um
def on_drop(window, count, paths):
nonlocal rec_dir
rec_dir = paths[0].decode("utf-8")
if rec_dir:
if not pm.is_pupil_rec_dir(rec_dir):
rec_dir = None
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player")
)
if VersionFormat(session_settings.get("version", "0.0")) != app_version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
w, h = session_settings.get("window_size", (1280, 720))
window_pos = session_settings.get("window_position", window_position_default)
glfw.glfwInit()
glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0)
window = glfw.glfwCreateWindow(w, h, "Pupil Player")
glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1)
glfw.glfwMakeContextCurrent(window)
glfw.glfwSetWindowPos(window, window_pos[0], window_pos[1])
glfw.glfwSetDropCallback(window, on_drop)
glfont = fontstash.Context()
glfont.add_font("roboto", get_roboto_font_path())
glfont.set_align_string(v_align="center", h_align="middle")
glfont.set_color_float((0.2, 0.2, 0.2, 0.9))
gl_utils.basic_gl_setup()
glClearColor(0.5, 0.5, 0.5, 0.0)
text = "Drop a recording directory onto this window."
tip = "(Tip: You can drop a recording directory onto the app icon.)"
# text = "Please supply a Pupil recording directory as first arg when calling Pupil Player."
while not glfw.glfwWindowShouldClose(window):
fb_size = glfw.glfwGetFramebufferSize(window)
hdpi_factor = glfw.getHDPIFactor(window)
gl_utils.adjust_gl_view(*fb_size)
if rec_dir:
if pm.is_pupil_rec_dir(rec_dir):
logger.info("Starting new session with '{}'".format(rec_dir))
text = "Updating recording format."
tip = "This may take a while!"
else:
logger.error("'{}' is not a valid pupil recording".format(rec_dir))
tip = "Oops! That was not a valid recording."
rec_dir = None
gl_utils.clear_gl_screen()
glfont.set_blur(10.5)
glfont.set_color_float((0.0, 0.0, 0.0, 1.0))
glfont.set_size(w / 25.0 * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, 0.3 * h * hdpi_factor, text)
glfont.set_size(w / 30.0 * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, 0.4 * h * hdpi_factor, tip)
glfont.set_blur(0.96)
glfont.set_color_float((1.0, 1.0, 1.0, 1.0))
glfont.set_size(w / 25.0 * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, 0.3 * h * hdpi_factor, text)
glfont.set_size(w / 30.0 * hdpi_factor)
glfont.draw_text(w / 2 * hdpi_factor, 0.4 * h * hdpi_factor, tip)
glfw.glfwSwapBuffers(window)
if rec_dir:
try:
um.update_recording_to_recent(rec_dir)
except AssertionError as err:
logger.error(str(err))
rec_dir = None
else:
glfw.glfwSetWindowShouldClose(window, True)
glfw.glfwPollEvents()
session_settings["window_position"] = glfw.glfwGetWindowPos(window)
session_settings.close()
glfw.glfwDestroyWindow(window)
if rec_dir:
ipc_pub.notify(
{"subject": "player_process.should_start", "rec_dir": rec_dir}
)
except:
import traceback
trace = traceback.format_exc()
logger.error("Process player_drop crashed with trace:\n{}".format(trace))
finally:
sleep(1.0)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version,
eye_id, cap_src):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
is_alive = Is_Alive_Manager(is_alive_flag)
with is_alive:
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id),
mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'Eye' + str(eye_id) +
' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'EYE' + str(eye_id) +
' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logging.getLogger("libav").setLevel(logging.ERROR)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture
from OpenGL.GL import GL_LINE_LOOP
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based
from ui_roi import UIRoi
#monitoring
import psutil
# helpers/utils
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from av_writer import JPEG_Writer, AV_Writer
# Pupil detectors
from pupil_detectors import Canny_Detector, Detector_2D, Detector_3D
pupil_detectors = {
Canny_Detector.__name__: Canny_Detector,
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D
}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos,
(frame.width, frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size + 40,
g_pool.u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version",
VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
# pipe_to_world.send('eye%s process ready'%eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode',
'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
}
g_pool.u_r = UIRoi(frame.img.shape)
g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1], new_size[0]))
cap.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,
pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s" % eye_id
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfw.glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector(
'pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Canny_Detector, Detector_2D, Detector_3D],
labels=[
'Python 2D detector', 'C++ 2d detector', 'C++ 3d detector'
],
label="Detection method")
general_settings.append(detector_selector)
# let detector add its GUI
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1 / 60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if pipe_to_world.poll():
cmd = pipe_to_world.recv()
if cmd == 'Exit':
break
elif cmd == "Ping":
pipe_to_world.send("Pong")
command = None
else:
command, payload = cmd
if command == 'Set_Detection_Mapping_Mode':
if payload == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
set_detector(Detector_2D)
detector_selector.read_only = False
else:
command = None
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
cap.seek_to_frame(0)
frame = cap.get_frame()
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if 'Rec_Start' == command:
record_path, raw_mode = payload
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(record_path,
"eye%s_timestamps.npy" % eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path, cap.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = AV_Writer(video_path, cap.frame_rate)
timestamps = []
elif 'Rec_Stop' == command:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
# switch to work in pixel space
make_coord_system_pixel_based(
(frame.height, frame.width, 3), g_pool.flip)
if result['confidence'] > 0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
draw_polyline(pts, 1, RGBA(1., 0, 0, .5))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., .5),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
g_pool.u_r.draw(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize(
) #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = g_pool.version
session_settings[
'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings[
'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
logger.debug("Process done")
def player(g_pool,size):
"""player
- Shows 9 point calibration pattern
- Plays a source video synchronized with world process
- Get src videos from directory (glob)
- Iterate through videos on each record event
"""
grid = make_grid()
grid *=2.5###scale to fit
# player object
player = Temp()
player.play_list = glob('src_video/*')
path_parent = os.path.dirname( os.path.abspath(sys.argv[0]))
player.playlist = [os.path.join(path_parent, path) for path in player.play_list]
player.captures = [autoCreateCapture(src) for src in player.playlist]
print "Player found %i videos in src_video"%len(player.captures)
player.captures = [c for c in player.captures if c is not None]
print "Player sucessfully loaded %i videos in src_video"%len(player.captures)
# for c in player.captures: c.auto_rewind = False
player.current_video = 0
# Callbacks
def on_resize(w, h):
adjust_gl_view(w,h)
def on_key(key, pressed):
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if pressed:
g_pool.player_input.value = char
def on_close():
g_pool.quit.value = True
print "Player Process closing from window"
def draw_circle(pos,r,c):
pts = cv2.ellipse2Poly(tuple(pos),(r,r),0,0,360,10)
draw_gl_polyline(pts,c,'Polygon')
def draw_marker(pos):
pos = int(pos[0]),int(pos[1])
black = (0.,0.,0.,1.)
white = (1.,1.,1.,1.)
for r,c in zip((50,40,30,20,10),(black,white,black,white,black)):
draw_circle(pos,r,c)
# Initialize glfw
glfwInit()
glfwOpenWindow(size[0], size[1], 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("Player")
glfwSetWindowPos(100,0)
glfwDisable(GLFW_AUTO_POLL_EVENTS)
# Callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
# gl state settings
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
gl.glDisable (gl.GL_DEPTH_TEST)
gl.glClearColor(1.,1.,1.,0.)
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
glfwPollEvents()
if g_pool.player_refresh.wait(0.01):
g_pool.player_refresh.clear()
clear_gl_screen()
if g_pool.marker_state.value !=0:
# Set Matrix unsing gluOrtho2D to include padding for the marker of radius r
#
############################
# r #
# 0,0##################w,h #
# # # #
# # # #
#r# #r#
# # # #
# # # #
# 0,h##################w,h #
# r #
############################
r = 60
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
# compensate for radius of marker
gluOrtho2D(-r,glfwGetWindowSize()[0]+r,glfwGetWindowSize()[1]+r, -r) # origin in the top left corner just like the img np-array
# Switch back to Model View Matrix
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
screen_pos = denormalize(g_pool.marker[:],glfwGetWindowSize(),flip_y=True)
#some feedback on the detection state
draw_marker(screen_pos)
if g_pool.ref[:] == [0.,0.]:
# world ref is detected
draw_gl_point(screen_pos, 5.0, (1.,0.,0.,1.))
else:
draw_gl_point(screen_pos, 5.0, (0.,1.,0.,1.))
elif g_pool.play.value:
if len(player.captures):
frame = player.captures[player.current_video].get_frame()
img = frame.img
if img:
draw_gl_texture(img)
else:
player.captures[player.current_video].rewind()
player.current_video +=1
if player.current_video >= len(player.captures):
player.current_video = 0
g_pool.play.value = False
else:
print 'PLAYER: Warning: No Videos available to play. Please put your vidoes into a folder called "src_video" in the Capture folder.'
g_pool.play.value = False
glfwSwapBuffers()
glfwCloseWindow()
glfwTerminate()
print "PLAYER Process closed"
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, version, eye_id, overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# general imports
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible
from ui_roi import UIRoi
# monitoring
import psutil
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, AV_Writer
from ndsi import H264Writer
from video_capture import source_classes
from video_capture import manager_classes
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D}
# UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 10.0
window_position_default = (600,31+ 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
# g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.process = 'eye{}'.format(eye_id)
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40):
# if the roi interacts we dont want
# the gui to interact as well
return
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = glfw.glfwGetFramebufferSize(
window)[0] / glfw.glfwGetWindowSize(window)[0]
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,g_pool.capture.frame_size )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode('utf-8') for x in range(count)]
g_pool.capture_manager.on_drop(paths)
g_pool.capture.on_drop(paths)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id)))
if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
logger.info("Session setting are from a different version of this app. I will not use those.")
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get(
'display_mode', 'camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager',{}))
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__:c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
if eye_id == 0:
cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
else:
cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]
# Initialize capture
default_settings = ('UVC_Source',{
'preferred_names' : cap_src,
'frame_size': (640,480),
'frame_rate': 90
})
capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
source_class_name, source_settings = capture_source_settings
source_class_by_name = {c.__name__:c for c in source_classes}
g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
assert g_pool.capture
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(
g_pool, pupil_detector_settings)
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "Pupil Capture - eye {}".format(eye_id)
width, height = session_settings.get(
'window_size', g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get(
'window_position', window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Selector('gui_user_scale', g_pool,
setter=set_scale,
selection=[.8, .9, 1., 1.1, 1.2],
label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image','roi','algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode")
)
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector, selection=[
Detector_2D, Detector_3D],
labels=['C++ 2d detector',
'C++ 3d detector'],
label="Detection method")
general_settings.append(detector_selector)
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture_source_menu.collapsed = True
g_pool.capture.init_gui()
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
g_pool.capture_manager.init_gui()
g_pool.writer = None
def replace_source(source_class_name,source_settings):
g_pool.capture.cleanup()
g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
g_pool.capture.init_gui()
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
g_pool.replace_source = replace_source # for ndsi capture
def replace_manager(manager_class):
g_pool.capture_manager.cleanup()
g_pool.capture_manager = manager_class(g_pool)
g_pool.capture_manager.init_gui()
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(0,ui.Selector(
'capture_manager',g_pool,
setter = replace_manager,
getter = lambda: g_pool.capture_manager.__class__,
selection = manager_classes,
labels = [b.gui_name for b in manager_classes],
label = 'Manager'
))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = 'jpeg'
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification['subject']
if subject.startswith('eye_process.should_stop'):
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye'] and g_pool.capture.online:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: {}".format(record_path))
video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id))
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate)
elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'):
g_pool.writer = H264Writer(video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate)
else:
g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate)
elif subject == 'recording.stopped':
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject': 'meta.doc',
'actor': 'eye{}'.format(eye_id),
'doc': eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format', 'jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process:
replace_source(notification['name'],notification['args'])
g_pool.capture.on_notify(notification)
# Get an image from the grabber
event = {}
g_pool.capture.recent_events(event)
frame = event.get('frame')
g_pool.capture_manager.recent_events(event)
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width):
g_pool.pupil_detector.on_resolution_change((g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size)
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
else:
raise AttributeError()
except AttributeError:
pass
else:
pupil_socket.send('frame.eye.%s'%eye_id,{
'width': frame.width,
'height': frame.height,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s'%eye_id,result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
if frame:
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
f_width, f_height = g_pool.capture.frame_size
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
if frame:
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly(
(int(eye_ball['center'][0]),
int(eye_ball['center'][1])),
(int(eye_ball['axes'][0] / 2),
int(eye_ball['axes'][1] / 2)),
int(eye_ball['angle']), 0, 360, 8)
except ValueError as e:
pass
else:
draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence']))
if result['confidence'] > 0:
if 'ellipse' in result:
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
confidence = result['confidence'] * 0.7
draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., confidence),
sharpness=1.)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
g_pool.gui.update()
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() # detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer = None
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict()
session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_gui()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
logger.info("Process shutting down.")
def eye(g_pool,cap_src,cap_size,rx_from_world,eye_id=0):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye%s.log'%eye_id),mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level+10)
# create formatter and add it to the handlers
formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor, h*hdpi_factor
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
# for p in g_pool.plugins:
# p.on_window_resize(window,w,h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == GLFW_RELEASE and u_r.active_edit_pt:
u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if u_r.mouse_over_edit_pt(pos,u_r.handle_size+40,u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if u_r.active_edit_pt:
pos = normalize((x,y),glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
u_r.move_vertex(u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
cap.frame_size = cap_size
cap.frame_rate = 90 #default
cap.settings = session_settings.get('capture_settings',{})
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
g_pool.capture = cap
g_pool.flip = session_settings.get('flip',False)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.window_size = session_settings.get('window_size',1.)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."}
# g_pool.draw_pupil = session_settings.get('draw_pupil',True)
u_r = UIRoi(frame.img.shape)
u_r.set(session_settings.get('roi',u_r.get()))
writer = None
pupil_detector = Canny_Detector(g_pool)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
window_pos = session_settings.get('window_position',window_position_default)
width,height = session_settings.get('window_size',(frame.width, frame.height))
# Initialize glfw
glfwInit()
if g_pool.binocular:
title = "Binocular eye %s"%eye_id
else:
title = 'Eye'
main_window = glfwCreateWindow(width,height, title, None, None)
glfwMakeContextCurrent(main_window)
cygl_init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex,frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# let detector add its GUI
pupil_detector.init_gui(g_pool.sidebar)
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if rx_from_world.poll():
command,raw_mode = rx_from_world.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s"%record_path)
video_path = os.path.join(record_path, "eye%s.mkv"%eye_id)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode:
writer = JPEG_Dumper(video_path)
else:
writer = CV_Writer(video_path,float(cap.frame_rate), cap.frame_size)
timestamps = []
else:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(frame,user_roi=u_r,visualize=g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
update_named_texture(g_pool.image_tex,frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
draw_named_texture(g_pool.image_tex)
# switch to work in pixel space
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
if result['confidence'] >0:
if result.has_key('axes'):
pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
(int(result['axes'][0]/2),int(result['axes'][1]/2)),
int(result['angle']),0,360,15)
cygl_draw_polyline(pts,1,cygl_rgba(1.,0,0,.5))
cygl_draw_points([result['center']],size=20,color=cygl_rgba(1.,0.,0.,.5),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
u_r.draw(g_pool.gui.scale)
#update screen
glfwSwapBuffers(main_window)
glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
pupil_detector.cleanup()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
#flushing queue in case world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
is_alive = Is_Alive_Manager(is_alive_flag)
with is_alive:
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level+10)
# create formatter and add it to the handlers
formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from av_writer import JPEG_Writer,AV_Writer
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':(640,480),'frame_rate':60}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
# pipe_to_world.send('eye%s process ready'%eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip',False)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
g_pool.u_r = UIRoi(frame.img.shape)
g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get()))
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1],new_size[0]))
cap.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s"%eye_id
width,height = session_settings.get('window_size',(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width,height, title, None, None)
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfw.glfwSwapInterval(0)
sphere = Sphere(20)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
general_settings.append(detector_selector)
# let detector add its GUI
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1/60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if pipe_to_world.poll():
cmd = pipe_to_world.recv()
if cmd == 'Exit':
break
elif cmd == "Ping":
pipe_to_world.send("Pong")
command = None
else:
command,payload = cmd
if command == 'Set_Detection_Mapping_Mode':
if payload == '3d':
if not isinstance(g_pool.pupil_detector,Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
set_detector(Detector_2D)
detector_selector.read_only = False
else:
command = None
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
cap.seek_to_frame(0)
frame = cap.get_frame()
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if 'Rec_Start' == command:
record_path,raw_mode = payload
logger.info("Will save eye video to: %s"%record_path)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = JPEG_Writer(video_path,cap.frame_rate)
else:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = AV_Writer(video_path,cap.frame_rate)
timestamps = []
elif 'Rec_Stop' == command:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
(int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
int(eye_ball['angle']),0,360,8)
except ValueError as e:
pass
else:
draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )
if result['confidence'] >0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
int(result['ellipse']['angle']),0,360,15)
confidence = result['confidence'] * 0.7 #scale it a little
draw_polyline(pts,1,RGBA(1.,0,0,confidence))
draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
g_pool.u_r.draw(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
logger.debug("Process done")
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, version, eye_id, glint_queue, glint_vector_queue, overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# general imports
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible
from ui_roi import UIRoi
# monitoring
import psutil
import math
from time import time
import json
# helpers/utils
from file_methods import Persistent_Dict, load_object
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, AV_Writer
from ndsi import H264Writer
from video_capture import source_classes
from video_capture import manager_classes
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D}
# UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 10.0
window_position_default = (600,31+ 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
# g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.process = 'eye{}'.format(eye_id)
g_pool.timebase = timebase
g_pool.glints = glint_queue
g_pool.glint_pupil_vectors = glint_vector_queue
glint_detector = Glint_Detector(g_pool)
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40):
# if the roi interacts we dont want
# the gui to interact as well
return
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = glfw.glfwGetFramebufferSize(
window)[0] / glfw.glfwGetWindowSize(window)[0]
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,g_pool.capture.frame_size )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id)))
if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
logger.info("Session setting are from a different version of this app. I will not use those.")
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get(
'display_mode', 'camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager',{}))
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__:c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)
g_pool.pupil_settings = "default"
if eye_id == 0:
cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
else:
cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]
# Initialize capture
default_settings = ('UVC_Source',{
'preferred_names' : cap_src,
'frame_size': (640,480),
'frame_rate': 90
})
capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
source_class_name, source_settings = capture_source_settings
source_class_by_name = {c.__name__:c for c in source_classes}
g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
assert g_pool.capture
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(
g_pool, pupil_detector_settings)
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
def convert_keys_to_string(dictionary):
"""Recursively converts dictionary keys to strings."""
if not isinstance(dictionary, dict):
return dictionary
return dict((str(k.decode('utf8')), convert_keys_to_string(v))
for k, v in dictionary.items())
def set_pupil_settings(new_settings):
g_pool.pupil_settings = new_settings
if not new_settings == "default":
try:
path = os.path.join(g_pool.user_dir,'pupil_settings_' + new_settings + '.json')
with open(path, 'r') as fp:
json_str = fp.read()
pupil_settings_new = json.loads(json_str)
except:
logger.error("Settings don't exist")
#pupil_settings_new = convert_keys_to_string(pupil_settings_new)
pupil_settings_new['2D_Settings'] = dict(pupil_settings_new)
pupil_settings = g_pool.pupil_detector.get_settings()
controls = g_pool.capture.uvc_capture.controls
controls_dict = dict([(c.display_name,c) for c in controls])
try:
g_pool.capture.frame_rate = pupil_settings_new['frame_rate']
g_pool.capture.frame_size = pupil_settings_new['frame_size']
except:
logger.info("no frame rate and frame size in camera settings")
for key in controls_dict:
print(key)
try:
controls_dict[key].value = pupil_settings_new[key]
except:
logger.info("no key with the name '%s' in camera settings" %key)
for key in pupil_settings.keys():
print(key)
try:
if type(pupil_settings[key]) == dict:
for sec_key in pupil_settings[key].keys():
pupil_settings[key][sec_key] = pupil_settings_new[key][sec_key]
else:
pupil_settings[key] = pupil_settings_new[key]
except:
logger.info("no key with the name '%s' in pupil settings" %key)
# Initialize glfw
glfw.glfwInit()
title = "Pupil Capture - eye {}".format(eye_id)
width, height = session_settings.get(
'window_size', g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get(
'window_position', window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Selector('pupil_settings',g_pool,setter=set_pupil_settings,selection=['default','indoors','outdoors_sunny', 'outdoors_cloudy', 'vanilla'], labels=['Default', 'Indoors', 'Outdoors Sunny', 'Outdoors Cloudy', 'Vanilla'], label="Pupil settings") )
general_settings.append(ui.Selector('gui_user_scale', g_pool,
setter=set_scale,
selection=[.8, .9, 1., 1.1, 1.2],
label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,*g_pool.capture.frame_size)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image','roi','algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode")
)
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector, selection=[
Detector_2D, Detector_3D],
labels=['C++ 2d detector',
'C++ 3d detector'],
label="Detection method")
general_settings.append(detector_selector)
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture_source_menu.collapsed = True
g_pool.capture.init_gui()
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
g_pool.capture_manager.init_gui()
g_pool.writer = None
def replace_source(source_class_name,source_settings):
g_pool.capture.cleanup()
g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
g_pool.capture.init_gui()
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
g_pool.replace_source = replace_source # for ndsi capture
def replace_manager(manager_class):
g_pool.capture_manager.cleanup()
g_pool.capture_manager = manager_class(g_pool)
g_pool.capture_manager.init_gui()
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(0,ui.Selector(
'capture_manager',g_pool,
setter = replace_manager,
getter = lambda: g_pool.capture_manager.__class__,
selection = manager_classes,
labels = [b.gui_name for b in manager_classes],
label = 'Manager'
))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = 'jpeg'
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification['subject']
if subject == 'eye_process.should_stop':
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye'] and g_pool.capture.online:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: {}".format(record_path))
video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id))
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate)
elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'):
g_pool.writer = H264Writer(video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate)
else:
g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate)
elif subject == 'recording.stopped':
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject': 'meta.doc',
'actor': 'eye{}'.format(eye_id),
'doc': eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format', 'jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process:
replace_source(notification['name'],notification['args'])
g_pool.capture.on_notify(notification)
# Get an image from the grabber
event = {}
g_pool.capture.recent_events(event)
frame = event.get('frame')
g_pool.capture_manager.recent_events(event)
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width):
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames and frame.jpeg_buffer:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
pupil_socket.send('frame.eye.%s'%eye_id,{
'width': frame.width,
'height': frame.width,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
t = frame.timestamp
tUnix = time()
dt, ts = t - ts, t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
# pupil ellipse detection
result, roi = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
result['unix_ts'] = tUnix
#glint detection
#glints = [[0,0,0,0,0,0], [0,0,0,0,0,1]] #glint_detector.glint(frame, eye_id, u_roi=g_pool.u_r, pupil=result, roi=roi)
#result['glints'] = glints
#g_pool.glints.put(glints)
#save glint-pupil vector results
#if glints[0][3]:
# glint_pupil_vector = {'timestamp': glints[0][0], 'x': result['norm_pos'][0]-glints[0][3], 'y': result['norm_pos'][1]-glints[0][4], 'pupil_confidence': result['confidence'], 'glint_found': True, 'id': eye_id, 'x2': result['norm_pos'][0]-glints[1][3], 'y2': result['norm_pos'][1]-glints[1][4]}
#else:
# glint_pupil_vector = {'timestamp': glints[0][0], 'x': result['norm_pos'][0]-glints[0][3], 'y': result['norm_pos'][1]-glints[0][4], 'pupil_confidence': result['confidence'], 'glint_found': False, 'id': eye_id, 'x2': result['norm_pos'][0]-glints[1][3], 'y2': result['norm_pos'][1]-glints[1][4]}
#g_pool.glint_pupil_vectors.put(glint_pupil_vector)
# stream the result
pupil_socket.send('pupil.%s'%eye_id,result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
if frame:
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
#if frame:
# glints = np.array(result['glints'])
# if len(glints)>0 and glints[0][3]:
# if glints[1][3]:
# cygl_draw_points(glints[:,1:3], size=20,color=cygl_rgba(0.,0.,1.,.5),sharpness=1.)
# elif result['confidence'] > 0.75:
# cygl_draw_points(glints[:,1:3], size=20,color=cygl_rgba(0.,0.,1.,.5),sharpness=1.)
f_width, f_height = g_pool.capture.frame_size
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
if frame:
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly(
(int(eye_ball['center'][0]),
int(eye_ball['center'][1])),
(int(eye_ball['axes'][0] / 2),
int(eye_ball['axes'][1] / 2)),
int(eye_ball['angle']), 0, 360, 8)
except ValueError as e:
pass
else:
draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence']))
if result['confidence'] > 0:
if 'ellipse' in result:
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
confidence = result['confidence'] * 0.7
draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., confidence),
sharpness=1.)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
g_pool.gui.update()
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() # detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer = None
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict()
session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_gui()
g_pool.pupil_detector.cleanup()
glint_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
logger.info("Process shutting down.")
