def gl_display_in_window(self,world_tex_id):
"""
here we map a selected surface onto a seperate window.
"""
if self._window and self.detected:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_from_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
glOrtho(0, 1, 0, 1,-1,1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
draw_named_texture(world_tex_id)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
# now lets get recent pupil positions on this surface:
draw_points_norm(self.gaze_on_srf,color=RGBA(0.,8.,.5,.8), size=80)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
if self.window_should_close:
self.close_window()
def gl_display_in_window_3d(self, world_tex_id, camera_intrinsics):
"""
here we map a selected surface onto a seperate window.
"""
K, dist_coef, img_size = camera_intrinsics
if self._window and self.detected:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
glClearColor(.8, .8, .8, 1.)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
self.trackball.push()
glMatrixMode(GL_MODELVIEW)
draw_coordinate_system(l=self.real_world_size['x'])
glPushMatrix()
glScalef(self.real_world_size['x'], self.real_world_size['y'], 1)
draw_gl_polyline([[0, 0], [0, 1], [1, 1], [1, 0]],
color=RGBA(.5, .3, .1, .5),
thickness=3)
glPopMatrix()
# Draw the world window as projected onto the plane using the homography mapping
glPushMatrix()
glScalef(self.real_world_size['x'], self.real_world_size['y'], 1)
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_from_screen)
glMultMatrixf(m)
glTranslatef(0, 0, -.01)
draw_named_texture(world_tex_id)
draw_gl_polyline([[0, 0], [0, 1], [1, 1], [1, 0]],
color=RGBA(.5, .3, .6, .5),
thickness=3)
glPopMatrix()
# Draw the camera frustum and origin using the 3d tranformation obtained from solvepnp
glPushMatrix()
glMultMatrixf(self.camera_pose_3d.T.flatten())
draw_frustum(self.img_size, K, 150)
glLineWidth(1)
draw_frustum(self.img_size, K, .1)
draw_coordinate_system(l=5)
glPopMatrix()
self.trackball.pop()
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window_3d(self,world_tex_id,camera_intrinsics):
"""
here we map a selected surface onto a seperate window.
"""
K,dist_coef,img_size = camera_intrinsics
if self._window and self.detected:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
glClearColor(.8,.8,.8,1.)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
self.trackball.push()
glMatrixMode(GL_MODELVIEW)
draw_coordinate_system(l=self.real_world_size['x'])
glPushMatrix()
glScalef(self.real_world_size['x'],self.real_world_size['y'],1)
draw_polyline([[0,0],[0,1],[1,1],[1,0]],color = RGBA(.5,.3,.1,.5),thickness=3)
glPopMatrix()
# Draw the world window as projected onto the plane using the homography mapping
glPushMatrix()
glScalef(self.real_world_size['x'], self.real_world_size['y'], 1)
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_from_screen)
glMultMatrixf(m)
glTranslatef(0,0,-.01)
draw_named_texture(world_tex_id)
draw_polyline([[0,0],[0,1],[1,1],[1,0]],color = RGBA(.5,.3,.6,.5),thickness=3)
glPopMatrix()
# Draw the camera frustum and origin using the 3d tranformation obtained from solvepnp
glPushMatrix()
glMultMatrixf(self.camera_pose_3d.T.flatten())
draw_frustum(self.img_size, K, 150)
glLineWidth(1)
draw_frustum(self.img_size, K, .1)
draw_coordinate_system(l=5)
glPopMatrix()
self.trackball.pop()
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_in_window(self, world_tex_id):
"""
here we map a selected surface onto a seperate window.
"""
if self._window and self.detected:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(self._window)
clear_gl_screen()
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_from_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
glOrtho(0, 1, 0, 1, -1, 1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
draw_named_texture(world_tex_id)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
if self.heatmap_texture:
draw_named_texture(self.heatmap_texture)
# now lets get recent pupil positions on this surface:
for gp in self.gaze_on_srf:
draw_points_norm([gp['norm_pos']],
color=RGBA(0.0, 0.8, 0.5, 0.8),
size=80)
glfwSwapBuffers(self._window)
glfwMakeContextCurrent(active_window)
def gl_display_metrics(self):
if self.metrics_texture and self.detected:
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_to_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
glOrtho(0, 1, 0, 1, -1, 1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
draw_named_texture(self.metrics_texture)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def gl_display_heatmap(self):
if self.heatmap_texture and self.detected:
# cv uses 3x3 gl uses 4x4 tranformation matricies
m = cvmat_to_glmat(self.m_to_screen)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
glOrtho(0, 1, 0, 1,-1,1) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
#apply m to our quad - this will stretch the quad such that the ref suface will span the window extends
glLoadMatrixf(m)
draw_named_texture(self.heatmap_texture)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex,frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
# fps_graph.draw()
# cpu_graph.draw()
# pupil_graph.draw()
# pupil_dia_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
def session(rec_dir):
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'" % new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'%s' is not a valid pupil recording" %
new_rec_dir)
video_path = glob(os.path.join(rec_dir, "world.*"))[0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
#parse info.csv file
meta_info_path = os.path.join(rec_dir, "info.csv")
with open(meta_info_path) as info:
meta_info = dict(
((line.strip().split('\t')) for line in info.readlines()))
rec_version = read_rec_version(meta_info)
if rec_version >= VersionFormat('0.5'):
pass
elif rec_version >= VersionFormat('0.4'):
update_recording_0v4_to_current(rec_dir)
elif rec_version >= VersionFormat('0.3'):
update_recording_0v3_to_current(rec_dir)
timestamps_path = os.path.join(rec_dir, "timestamps.npy")
else:
logger.Error("This recording is to old. Sorry.")
return
# Initialize capture
cap = autoCreateCapture(video_path, timestamps=timestamps_path)
if isinstance(cap, FakeCapture):
logger.error("could not start capture.")
return
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version",
VersionFormat('0.0')) < get_version(version_file):
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', (0, 0))
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = np.load(timestamps_path)
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
] #populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s' % plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Growing_Menu("Settings",
pos=(-350, 20),
size=(300, 400))
g_pool.main_menu.append(ui.Button("quit", lambda: on_close(None)))
g_pool.main_menu.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=0.75,
max=2.5,
label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
rec_version))
g_pool.main_menu.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label='Play',
hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend(
[g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Scan_Path', {}), ('Vis_Polyline', {}),
('Vis_Circle', {}), ('Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
#trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height, width, 3))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now() - .03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(
g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(
g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex, frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
g_pool.gui.terminate()
destroy_named_texture(g_pool.image_tex)
glfwDestroyWindow(main_window)
def world(g_pool,cap_src,cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
#manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins
system_plugins = [Log_Display,Display_Recent_Gaze,Recorder]
plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index,plugin_by_index))
default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':cap_size,'frame_rate':24}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = session_settings.get("update_textures",2)
g_pool.iconified = False
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
g_pool.active_calibration_plugin = None
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
#window and gl setup
glfwInit()
width,height = session_settings.get('window_size',(frame.width, frame.height))
main_window = glfwCreateWindow(width,height, "World")
window_pos = session_settings.get('window_position',window_position_default)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.append(ui.Selector('update_textures',g_pool,label="Update display",selection=range(3),labels=('No update','Gray','Color')))
advanced_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
advanced_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.append(ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= open_plugin,label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetWindowIconifyCallback(main_window,on_iconify)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex,frame.img)
# refresh speed settings
glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
elif g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex,frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['update_textures'] = g_pool.update_textures
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
logger.debug("Process done")
def eye(g_pool,cap_src,cap_size,rx_from_world,eye_id=0):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye%s.log'%eye_id),mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level+10)
# create formatter and add it to the handlers
formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor, h*hdpi_factor
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
# for p in g_pool.plugins:
# p.on_window_resize(window,w,h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == GLFW_RELEASE and u_r.active_edit_pt:
u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if u_r.mouse_over_edit_pt(pos,u_r.handle_size+40,u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if u_r.active_edit_pt:
pos = normalize((x,y),glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
u_r.move_vertex(u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
cap.frame_size = cap_size
cap.frame_rate = 90 #default
cap.settings = session_settings.get('capture_settings',{})
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
g_pool.capture = cap
g_pool.flip = session_settings.get('flip',False)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.window_size = session_settings.get('window_size',1.)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."}
# g_pool.draw_pupil = session_settings.get('draw_pupil',True)
u_r = UIRoi(frame.img.shape)
u_r.set(session_settings.get('roi',u_r.get()))
writer = None
pupil_detector = Canny_Detector(g_pool)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
window_pos = session_settings.get('window_position',window_position_default)
width,height = session_settings.get('window_size',(frame.width, frame.height))
# Initialize glfw
glfwInit()
if g_pool.binocular:
title = "Binocular eye %s"%eye_id
else:
title = 'Eye'
main_window = glfwCreateWindow(width,height, title, None, None)
glfwMakeContextCurrent(main_window)
cygl_init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex,frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# let detector add its GUI
pupil_detector.init_gui(g_pool.sidebar)
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if rx_from_world.poll():
command,raw_mode = rx_from_world.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s"%record_path)
video_path = os.path.join(record_path, "eye%s.mkv"%eye_id)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode:
writer = JPEG_Dumper(video_path)
else:
writer = CV_Writer(video_path,float(cap.frame_rate), cap.frame_size)
timestamps = []
else:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(frame,user_roi=u_r,visualize=g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
update_named_texture(g_pool.image_tex,frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
draw_named_texture(g_pool.image_tex)
# switch to work in pixel space
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
if result['confidence'] >0:
if result.has_key('axes'):
pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
(int(result['axes'][0]/2),int(result['axes'][1]/2)),
int(result['angle']),0,360,15)
cygl_draw_polyline(pts,1,cygl_rgba(1.,0,0,.5))
cygl_draw_points([result['center']],size=20,color=cygl_rgba(1.,0.,0.,.5),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
u_r.draw(g_pool.gui.scale)
#update screen
glfwSwapBuffers(main_window)
glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
pupil_detector.cleanup()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
#flushing queue in case world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def world(g_pool, cap_src, cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(
window)[0]
w, h = w * hdpi_factor, h * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
glfwMakeContextCurrent(active_window)
def on_iconify(window, iconfied):
if not isinstance(cap, FakeCapture):
g_pool.update_textures = not iconfied
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = 2
if isinstance(cap, FakeCapture):
g_pool.update_textures = 0
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get(
'pupil_confidence_threshold', .6)
g_pool.active_calibration_plugin = None
#UI callback functions
def reset_timebase():
#the last frame from worldcam will be t0
g_pool.timebase.value = g_pool.capture.get_now()
logger.info(
"New timebase set to %s all timestamps will count from here now." %
g_pool.timebase.value)
def set_calibration_plugin(new_calibration):
g_pool.active_calibration_plugin = new_calibration
new_plugin = new_calibration(g_pool)
g_pool.plugins.add(new_plugin)
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s' % plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
width, height = session_settings.get('window_size',
(frame.width, frame.height))
window_pos = session_settings.get('window_position',
(0, 0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(width, height, "World", None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetWindowIconifyCallback(main_window, on_iconify)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex, frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window, 0, 0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-250, 0),
size=(0, 0),
header_pos='left')
g_pool.sidebar.configuration = session_settings.get('side_bar_config', {})
general_settings = ui.Growing_Menu('General')
general_settings.configuration = session_settings.get(
'general_menu_config', {})
general_settings.append(
ui.Slider('scale',
setter=set_scale,
getter=get_scale,
step=.05,
min=1.,
max=2.5,
label='Interface size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.configuration = session_settings.get(
'advanced_menu_config', {'collapsed': True})
advanced_settings.append(
ui.Selector('update_textures',
g_pool,
label="Update display",
selection=range(3),
labels=('No update', 'Gray', 'Color')))
advanced_settings.append(
ui.Slider('pupil_confidence_threshold',
g_pool,
step=.01,
min=0.,
max=1.,
label='Minimum pupil confidence'))
advanced_settings.append(ui.Button('Set timebase to 0', reset_timebase))
advanced_settings.append(
ui.Info_Text('Capture Version: %s' % g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.configuration = session_settings.get(
'calibration_menu_config', {})
g_pool.calibration_menu.append(
ui.Selector(
'active_calibration_plugin',
g_pool,
selection=calibration_plugins,
labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
setter=set_calibration_plugin,
label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(
ui.Hot_Key("quit",
setter=on_close,
getter=lambda: True,
label="X",
hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.menu.configuration = session_settings.get(
'capture_menu_config', {})
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
#only needed for the gui to show the loaded calibration type
for p in g_pool.plugins:
if p.base_class_name == 'Calibration_Plugin':
g_pool.active_calibration_plugin = p.__class__
break
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
fps_graph.add(1. / dt)
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex, frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex, frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings[
'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['side_bar_config'] = g_pool.sidebar.configuration
session_settings['capture_menu_config'] = g_pool.capture.menu.configuration
session_settings['general_menu_config'] = general_settings.configuration
session_settings['advanced_menu_config'] = advanced_settings.configuration
session_settings[
'calibration_menu_config'] = g_pool.calibration_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def world(g_pool, cap_src, cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
#manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, 'plugins'))
user_launchable_plugins = [
Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector
] + runtime_plugins
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
('Display_Recent_Gaze', {}),
('Screen_Marker_Calibration', {}), ('Recorder', {})]
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconfied):
pass
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
cap.frame_size = cap_size
cap.frame_rate = 24 #default
cap.settings = session_settings.get('capture_settings', {})
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = session_settings.get("update_textures", 2)
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get(
'pupil_confidence_threshold', .6)
g_pool.active_calibration_plugin = None
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
#window and gl setup
glfwInit()
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfwCreateWindow(width, height, "World")
window_pos = session_settings.get('window_position',
window_position_default)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-350, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.append(
ui.Selector('update_textures',
g_pool,
label="Update display",
selection=range(3),
labels=('No update', 'Gray', 'Color')))
advanced_settings.append(
ui.Slider('pupil_confidence_threshold',
g_pool,
step=.01,
min=0.,
max=1.,
label='Minimum pupil confidence'))
advanced_settings.append(
ui.Info_Text('Capture Version: %s' % g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.append(
ui.Selector(
'active_calibration_plugin',
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection=calibration_plugins,
labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
setter=open_plugin,
label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(
ui.Hot_Key("quit",
setter=on_close,
getter=lambda: True,
label="X",
hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetWindowIconifyCallback(main_window, on_iconify)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex, frame.img)
# refresh speed settings
glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex, frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex, frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings[
'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['update_textures'] = g_pool.update_textures
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
logger.debug("Process done")
def main():
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor, h*hdpi_factor
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfwMakeContextCurrent(active_window)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*y_scroll_factor)
def on_close(window):
glfwSetWindowShouldClose(main_window,True)
logger.debug('Process closing from window')
try:
rec_dir = sys.argv[1]
except:
#for dev, supply hardcoded dir:
rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
if os.path.isdir(rec_dir):
logger.debug("Dev option: Using hadcoded data dir.")
else:
if getattr(sys, 'frozen', False):
logger.warning("You did not supply a data directory when you called this script! \
\nPlease drag a Pupil recoding directory onto the launch icon.")
else:
logger.warning("You did not supply a data directory when you called this script! \
\nPlease supply a Pupil recoding directory as first arg when calling Pupil Player.")
return
if not is_pupil_rec_dir(rec_dir):
logger.error("You did not supply a dir with the required files inside.")
return
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
#backwards compatibility fn.
patch_meta_info(rec_dir)
#parse info.csv file
meta_info_path = rec_dir + "/info.csv"
with open(meta_info_path) as info:
meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) )
rec_version = read_rec_version(meta_info)
if rec_version < VersionFormat('0.4'):
video_path = rec_dir + "/world.avi"
timestamps_path = rec_dir + "/timestamps.npy"
else:
video_path = rec_dir + "/world.mkv"
timestamps_path = rec_dir + "/world_timestamps.npy"
gaze_positions_path = rec_dir + "/gaze_positions.npy"
#load gaze information
gaze_list = np.load(gaze_positions_path)
timestamps = np.load(timestamps_path)
#correlate data
if rec_version < VersionFormat('0.4'):
positions_by_frame = correlate_gaze_legacy(gaze_list,timestamps)
else:
positions_by_frame = correlate_gaze(gaze_list,timestamps)
# Initialize capture
cap = autoCreateCapture(video_path,timestamps=timestamps_path)
if isinstance(cap,FakeCapture):
logger.error("could not start capture.")
return
width,height = session_settings.get('window_size',cap.get_size())
window_pos = session_settings.get('window_position',(0,0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# create container for globally scoped vars (within world)
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.gaze_list = gaze_list
g_pool.positions_by_frame = positions_by_frame
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s'%plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive=False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.main_menu = ui.Scrolling_Menu("Settings",pos=(-350,20),size=(300,300))
g_pool.main_menu.append(ui.Button("quit",lambda: on_close(None)))
g_pool.main_menu.configuration = session_settings.get('main_menu_config',{})
g_pool.main_menu.append(ui.Slider('scale', setter=set_scale,getter=get_scale,step = .05,min=0.75,max=2.5,label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version))
g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter = lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins))
g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.get_size()[0],cap.get_size()[1])) )
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0,1.,.0,.8)
g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})]
default_plugins = [('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Export_Launcher',{})]
previous_plugins = session_settings.get('loaded_plugins',default_plugins)
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
glfwSetWindowPos(main_window,0,0)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height,width,3))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now()-.03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,110)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play=False
if g_pool.new_seek:
display_time = new_frame.timestamp
g_pool.new_seek = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#new positons we make a deepcopy just like the image is a copy.
events['pupil_positions'] = deepcopy(positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt,ts = t-ts,t
fps_graph.add(1./dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex,frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time()-timestamp
sleep(wait_time-spent_time)
except:
pass
timestamp = time()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['main_menu_config'] = g_pool.main_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def eye(g_pool, cap_src, cap_size, rx_from_world, eye_id=0):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,
'eye%s.log' % eye_id),
mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'Eye' + str(eye_id) +
' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'EYE' + str(eye_id) +
' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(
window)[0]
w, h = w * hdpi_factor, h * hdpi_factor
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
# for p in g_pool.plugins:
# p.on_window_resize(window,w,h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == GLFW_RELEASE and u_r.active_edit_pt:
u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos, (frame.width, frame.height)) # Position in img pixels
if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if u_r.active_edit_pt:
pos = normalize((x, y), glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
u_r.move_vertex(u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size, 30, timebase=g_pool.timebase)
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.window_size = session_settings.get('window_size', 1.)
g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
}
# g_pool.draw_pupil = session_settings.get('draw_pupil',True)
u_r = UIRoi(frame.img.shape)
u_r.set(session_settings.get('roi', u_r.get()))
writer = None
pupil_detector = Canny_Detector(g_pool)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
def set_display_mode_info(val):
# set info text here and append to the general settings menu
# 'camera_image','roi','algorithm','cpu_save'
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
width, height = session_settings.get('window_size',
(frame.width, frame.height))
window_pos = session_settings.get('window_position',
(0, 0)) # not yet using this one.
# Initialize glfw
glfwInit()
if g_pool.binocular:
title = "Binocular eye %s" % eye_id
else:
title = 'Eye'
main_window = glfwCreateWindow(width, height, title, None, None)
glfwMakeContextCurrent(main_window)
cygl_init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex, frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window, 800, 300 * eye_id)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
g_pool.sidebar.configuration = session_settings.get(
'side_bar_config', {'collapsed': True})
general_settings = ui.Growing_Menu('General')
general_settings.configuration = session_settings.get(
'general_menu_config', {})
general_settings.append(
ui.Slider('scale',
setter=set_scale,
getter=get_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(
ui.Hot_Key("quit",
setter=on_close,
getter=lambda: True,
label="X",
hotkey=GLFW_KEY_ESCAPE))
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.menu.configuration = session_settings.get(
'capture_menu_config', {'collapsed': True})
# let detector add its GUI
pupil_detector.init_gui(g_pool.sidebar)
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if rx_from_world.poll():
command = rx_from_world.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s" % record_path)
video_path = os.path.join(record_path, "eye%s.mkv" % eye_id)
timestamps_path = os.path.join(record_path,
"eye%s_timestamps.npy" % eye_id)
writer = cv2.VideoWriter(
video_path, cv2.cv.CV_FOURCC(*'DIVX'),
float(cap.frame_rate),
(frame.img.shape[1], frame.img.shape[0]))
timestamps = []
else:
logger.info("Done recording.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write(frame.img)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(
frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
update_named_texture(g_pool.image_tex, frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
update_named_texture(g_pool.image_tex, frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
draw_named_texture(g_pool.image_tex)
# switch to work in pixel space
make_coord_system_pixel_based((frame.height, frame.width, 3),
g_pool.flip)
if result['confidence'] > 0:
if result.has_key('axes'):
pts = cv2.ellipse2Poly(
(int(result['center'][0]), int(result['center'][1])),
(int(result['axes'][0] / 2), int(result['axes'][1] / 2)),
int(result['angle']), 0, 360, 15)
cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
cygl_draw_points([result['center']],
size=20,
color=cygl_rgba(1., 0., 0., .5),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
u_r.draw(g_pool.gui.scale)
#update screen
glfwSwapBuffers(main_window)
glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['side_bar_config'] = g_pool.sidebar.configuration
session_settings['capture_menu_config'] = g_pool.capture.menu.configuration
session_settings['general_menu_config'] = general_settings.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
pupil_detector.cleanup()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
#flushing queue in case world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def session(rec_dir):
# Callback functions
def on_resize(window,w, h):
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*y_scroll_factor)
def on_drop(window,count,paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'"%new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window,True)
else:
logger.error("'%s' is not a valid pupil recording"%new_rec_dir)
tick = delta_t()
def get_dt():
return next(tick)
video_path = glob(os.path.join(rec_dir,"world.*"))[0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
#parse info.csv file
meta_info_path = os.path.join(rec_dir,"info.csv")
with open(meta_info_path) as info:
meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) )
rec_version = read_rec_version(meta_info)
if rec_version >= VersionFormat('0.5'):
pass
elif rec_version >= VersionFormat('0.4'):
update_recording_0v4_to_current(rec_dir)
elif rec_version >= VersionFormat('0.3'):
update_recording_0v3_to_current(rec_dir)
timestamps_path = os.path.join(rec_dir, "timestamps.npy")
else:
logger.Error("This recording is to old. Sorry.")
return
# Initialize capture
cap = File_Capture(video_path,timestamps=timestamps_path)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file):
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
width,height = session_settings.get('window_size',cap.frame_size)
window_pos = session_settings.get('window_position',(0,0))
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = np.load(timestamps_path)
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.pupil_positions_by_frame = correlate_data(pupil_list,g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list,g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] #populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.main_menu = ui.Growing_Menu("Settings",pos=(-350,20),size=(300,400))
g_pool.main_menu.append(ui.Button("Close Pupil Player",lambda:glfwSetWindowShouldClose(main_window,True)))
g_pool.main_menu.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=0.75,max=2.5,label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version))
g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter = lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins))
g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.frame_size[0],cap.frame_size[1])) )
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0,1.,.0,.8)
g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})]
default_plugins = [('Log_Display',{}),('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Export_Launcher',{})]
previous_plugins = session_settings.get('loaded_plugins',default_plugins)
g_pool.notifications = []
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window,on_resize)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
glfwSetDropCallback(main_window,on_drop)
#trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config',{})
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height,width,3))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now()-.03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play=False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt,ts = t-ts,t
fps_graph.add(1./dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex,frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
g_pool.gui.terminate()
destroy_named_texture(g_pool.image_tex)
glfwDestroyWindow(main_window)
def main():
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(
window)[0]
w, h = w * hdpi_factor, h * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfwMakeContextCurrent(active_window)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_close(window):
glfwSetWindowShouldClose(main_window, True)
logger.debug('Process closing from window')
try:
rec_dir = sys.argv[1]
except:
#for dev, supply hardcoded dir:
rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
if os.path.isdir(rec_dir):
logger.debug("Dev option: Using hadcoded data dir.")
else:
if getattr(sys, 'frozen', False):
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease drag a Pupil recoding directory onto the launch icon."
)
else:
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease supply a Pupil recoding directory as first arg when calling Pupil Player."
)
return
if not is_pupil_rec_dir(rec_dir):
logger.error(
"You did not supply a dir with the required files inside.")
return
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
#backwards compatibility fn.
patch_meta_info(rec_dir)
#parse info.csv file
meta_info_path = rec_dir + "/info.csv"
with open(meta_info_path) as info:
meta_info = dict(
((line.strip().split('\t')) for line in info.readlines()))
rec_version = read_rec_version(meta_info)
if rec_version < VersionFormat('0.4'):
video_path = rec_dir + "/world.avi"
timestamps_path = rec_dir + "/timestamps.npy"
else:
video_path = rec_dir + "/world.mkv"
timestamps_path = rec_dir + "/world_timestamps.npy"
gaze_positions_path = rec_dir + "/gaze_positions.npy"
#load gaze information
gaze_list = np.load(gaze_positions_path)
timestamps = np.load(timestamps_path)
#correlate data
if rec_version < VersionFormat('0.4'):
positions_by_frame = correlate_gaze_legacy(gaze_list, timestamps)
else:
positions_by_frame = correlate_gaze(gaze_list, timestamps)
# Initialize capture
cap = autoCreateCapture(video_path, timestamps=timestamps_path)
if isinstance(cap, FakeCapture):
logger.error("could not start capture.")
return
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position',
(0, 0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# create container for globally scoped vars (within world)
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.gaze_list = gaze_list
g_pool.positions_by_frame = positions_by_frame
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s' % plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Scrolling_Menu("Settings",
pos=(-350, 20),
size=(300, 300))
g_pool.main_menu.append(ui.Button("quit", lambda: on_close(None)))
g_pool.main_menu.configuration = session_settings.get(
'main_menu_config', {})
g_pool.main_menu.append(
ui.Slider('scale',
setter=set_scale,
getter=get_scale,
step=.05,
min=0.75,
max=2.5,
label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
rec_version))
g_pool.main_menu.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label='Play',
hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend(
[g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Scan_Path', {}), ('Vis_Polyline', {}),
('Vis_Circle', {}), ('Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
glfwSetWindowPos(main_window, 0, 0)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height, width, 3))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now() - .03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play = False
if g_pool.new_seek:
display_time = new_frame.timestamp
g_pool.new_seek = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#new positons we make a deepcopy just like the image is a copy.
events['pupil_positions'] = deepcopy(positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex, frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time() - timestamp
sleep(wait_time - spent_time)
except:
pass
timestamp = time()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['main_menu_config'] = g_pool.main_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def world(g_pool,cap_src,cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor, h*hdpi_factor
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
glfwMakeContextCurrent(active_window)
def on_iconify(window,iconfied):
pass
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = session_settings.get("update_textures",2)
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
g_pool.active_calibration_plugin = None
#UI callback functions
def reset_timebase():
#the last frame from worldcam will be t0
g_pool.timebase.value = g_pool.capture.get_now()
logger.info("New timebase set to %s all timestamps will count from here now."%g_pool.timebase.value)
def set_calibration_plugin(new_calibration):
g_pool.active_calibration_plugin = new_calibration
new_plugin = new_calibration(g_pool)
g_pool.plugins.add(new_plugin)
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s'%plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
width,height = session_settings.get('window_size',(frame.width, frame.height))
window_pos = session_settings.get('window_position',(0,0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(width,height, "World", None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetWindowIconifyCallback(main_window,on_iconify)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex,frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window,0,0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-250,0),size=(0,0),header_pos='left')
g_pool.sidebar.configuration = session_settings.get('side_bar_config',{})
general_settings = ui.Growing_Menu('General')
general_settings.configuration = session_settings.get('general_menu_config',{})
general_settings.append(ui.Slider('scale', setter=set_scale,getter=get_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.configuration = session_settings.get('advanced_menu_config',{'collapsed':True})
advanced_settings.append(ui.Selector('update_textures',g_pool,label="Update display",selection=range(3),labels=('No update','Gray','Color')))
advanced_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
advanced_settings.append(ui.Button('Set timebase to 0',reset_timebase))
advanced_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.configuration = session_settings.get('calibration_menu_config',{})
g_pool.calibration_menu.append(ui.Selector('active_calibration_plugin',g_pool, selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= set_calibration_plugin,label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.menu.configuration = session_settings.get('capture_menu_config',{})
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
#only needed for the gui to show the loaded calibration type
for p in g_pool.plugins:
if p.base_class_name == 'Calibration_Plugin':
g_pool.active_calibration_plugin = p.__class__
break
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex,frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['side_bar_config'] = g_pool.sidebar.configuration
session_settings['capture_menu_config'] = g_pool.capture.menu.configuration
session_settings['general_menu_config'] = general_settings.configuration
session_settings['advanced_menu_config'] = advanced_settings.configuration
session_settings['calibration_menu_config']=g_pool.calibration_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['update_textures'] = g_pool.update_textures
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
