def world(pupil_queue, timebase, lauchner_pipe, eye_pipes, eyes_are_alive, user_dir, version, cap_src):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir, "world.log"), mode="w")
fh.setLevel(logger.level)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter("World Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s")
fh.setFormatter(formatter)
formatter = logging.Formatter("WORLD Process [%(levelname)s] %(name)s : %(message)s")
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logging.getLogger("libav").setLevel(logging.ERROR)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmfull but unnessasary.
# general imports
from time import time
import numpy as np
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
from gl_utils import (
basic_gl_setup,
adjust_gl_view,
clear_gl_screen,
make_coord_system_pixel_based,
make_coord_system_norm_based,
)
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
assert pyglui_version >= "0.7"
# monitoring
import psutil
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from version_utils import VersionFormat
# Plug-ins
from plugin import Plugin_List, import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from recorder import Recorder
from show_calibration import Show_Calibration
from display_recent_gaze import Display_Recent_Gaze
from pupil_server import Pupil_Server
from pupil_sync import Pupil_Sync
from marker_detector import Marker_Detector
from log_display import Log_Display
from annotations import Annotation_Capture
# create logger for the context of this function
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (0, 0)
elif platform.system() == "Windows":
scroll_factor = 1.0
window_position_default = (8, 31)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
# g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = "capture"
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# g_pool.lauchner_pipe = lauchner_pipe
g_pool.eye_pipes = eye_pipes
g_pool.eyes_are_alive = eyes_are_alive
# manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, "plugins"))
user_launchable_plugins = [
Show_Calibration,
Pupil_Server,
Pupil_Sync,
Marker_Detector,
Annotation_Capture,
] + runtime_plugins
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [
("Log_Display", {}),
("Dummy_Gaze_Mapper", {}),
("Display_Recent_Gaze", {}),
("Screen_Marker_Calibration", {}),
("Recorder", {}),
]
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, "user_settings_world"))
if session_settings.get("version", VersionFormat("0.0")) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {"frame_size": (1280, 720), "frame_rate": 30}
previous_settings = session_settings.get("capture_settings", None)
if previous_settings and previous_settings["name"] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
lauchner_pipe.send("Exit")
return
g_pool.iconified = False
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get("pupil_confidence_threshold", 0.6)
g_pool.detection_mapping_mode = session_settings.get("detection_mapping_mode", "2d")
g_pool.active_calibration_plugin = None
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def launch_eye_process(eye_id, blocking=False):
if eyes_are_alive[eye_id].value:
logger.error("Eye%s process already running." % eye_id)
return
lauchner_pipe.send(eye_id)
eye_pipes[eye_id].send(("Set_Detection_Mapping_Mode", g_pool.detection_mapping_mode))
if blocking:
# wait for ready message from eye to sequentialize startup
eye_pipes[eye_id].send("Ping")
eye_pipes[eye_id].recv()
logger.warning("Eye %s process started." % eye_id)
def stop_eye_process(eye_id, blocking=False):
if eyes_are_alive[eye_id].value:
eye_pipes[eye_id].send("Exit")
if blocking:
raise NotImplementedError()
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
if new_mode == "2d":
for p in g_pool.plugins:
if "Vector_Gaze_Mapper" in p.class_name:
logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
p.alive = False
g_pool.plugins.clean()
for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes):
if alive.value:
pipe.send(("Set_Detection_Mapping_Mode", new_mode))
g_pool.detection_mapping_mode = new_mode
# window and gl setup
glfw.glfwInit()
width, height = session_settings.get("window_size", (frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width, height, "World")
window_pos = session_settings.get("window_position", window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get("gui_scale", 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos="left")
general_settings = ui.Growing_Menu("General")
general_settings.append(
ui.Slider("scale", g_pool.gui, setter=set_scale, step=0.05, min=1.0, max=2.5, label="Interface size")
)
general_settings.append(
ui.Button("Reset window size", lambda: glfw.glfwSetWindowSize(main_window, frame.width, frame.height))
)
general_settings.append(
ui.Selector(
"detection_mapping_mode",
g_pool,
label="detection & mapping mode",
setter=set_detection_mapping_mode,
selection=["2d", "3d"],
)
)
general_settings.append(
ui.Switch(
"eye0_process",
label="detect eye 0",
setter=lambda alive: start_stop_eye(0, alive),
getter=lambda: eyes_are_alive[0].value,
)
)
general_settings.append(
ui.Switch(
"eye1_process",
label="detect eye 1",
setter=lambda alive: start_stop_eye(1, alive),
getter=lambda: eyes_are_alive[1].value,
)
)
general_settings.append(
ui.Selector(
"Open plugin",
selection=user_launchable_plugins,
labels=[p.__name__.replace("_", " ") for p in user_launchable_plugins],
setter=open_plugin,
getter=lambda: "Select to load",
)
)
general_settings.append(
ui.Slider("pupil_confidence_threshold", g_pool, step=0.01, min=0.0, max=1.0, label="Minimum pupil confidence")
)
general_settings.append(ui.Info_Text("Capture Version: %s" % g_pool.version))
g_pool.sidebar.append(general_settings)
g_pool.calibration_menu = ui.Growing_Menu("Calibration")
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100))
g_pool.gui.append(g_pool.quickbar)
g_pool.capture.init_gui(g_pool.sidebar)
# plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get("loaded_plugins", default_plugins))
# We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(
0,
ui.Selector(
"active_calibration_plugin",
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection=calibration_plugins,
labels=[p.__name__.replace("_", " ") for p in calibration_plugins],
setter=open_plugin,
label="Method",
),
)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
# refresh speed settings
glfw.glfwSwapInterval(0)
# trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
if session_settings.get("eye1_process_alive", False):
launch_eye_process(1, blocking=True)
if session_settings.get("eye0_process_alive", True):
launch_eye_process(0, blocking=False)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
# update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1.0 / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
# a dictionary that allows plugins to post and read events
events = {}
# report time between now and the last loop interation
events["dt"] = get_dt()
# receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p["confidence"])
events["pupil_positions"] = recent_pupil_positions
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n["_notify_time_"] < time():
del n["_notify_time_"]
del g_pool.delayed_notifications[n["subject"]]
g_pool.notifications.append(n)
# notify each plugin if there are new notifications:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfw.glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
else:
g_pool.image_tex.update_from_frame(frame)
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["pupil_confidence_threshold"] = g_pool.pupil_confidence_threshold
session_settings["gui_scale"] = g_pool.gui.scale
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["capture_settings"] = g_pool.capture.settings
session_settings["window_size"] = glfw.glfwGetWindowSize(main_window)
session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
session_settings["version"] = g_pool.version
session_settings["eye0_process_alive"] = eyes_are_alive[0].value
session_settings["eye1_process_alive"] = eyes_are_alive[1].value
session_settings["detection_mapping_mode"] = g_pool.detection_mapping_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
# shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
# shut down laucher
lauchner_pipe.send("Exit")
logger.debug("world process done")
def eye(g_pool, cap_src, cap_size, pipe_to_world, eye_id=0):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,
'eye%s.log' % eye_id),
mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'Eye' + str(eye_id) +
' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'EYE' + str(eye_id) +
' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == GLFW_RELEASE and u_r.active_edit_pt:
u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos, (frame.width, frame.height)) # Position in img pixels
if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if u_r.active_edit_pt:
pos = normalize((x, y), glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
u_r.move_vertex(u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': cap_size, 'frame_rate': 30}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
pipe_to_world.send('eye%s process ready' % eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
}
# g_pool.draw_pupil = session_settings.get('draw_pupil',True)
u_r = UIRoi(frame.img.shape)
u_r.set(session_settings.get('roi', u_r.get()))
writer = None
pupil_detector = Canny_Detector(g_pool)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
# Initialize glfw
glfwInit()
if g_pool.binocular:
title = "Binocular eye %s" % eye_id
else:
title = 'Eye'
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl_init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# let detector add its GUI
pupil_detector.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetWindowIconifyCallback(main_window, on_iconify)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1 / 60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if pipe_to_world.poll():
command, raw_mode = pipe_to_world.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(record_path,
"eye%s_timestamps.npy" % eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path, cap.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = AV_Writer(video_path, cap.frame_rate)
timestamps = []
else:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(
frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
# switch to work in pixel space
make_coord_system_pixel_based((frame.height, frame.width, 3),
g_pool.flip)
if result['confidence'] > 0:
if result.has_key('axes'):
pts = cv2.ellipse2Poly((int(
result['center'][0]), int(result['center'][1])),
(int(result['axes'][0] / 2),
int(result['axes'][1] / 2)),
int(result['angle']), 0, 360,
15)
cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
cygl_draw_points([result['center']],
size=20,
color=cygl_rgba(1., 0., 0., .5),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
u_r.draw(g_pool.gui.scale)
#update screen
glfwSwapBuffers(main_window)
glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
pupil_detector.cleanup()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
#flushing queue in case world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def session(rec_dir):
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'" % new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'%s' is not a valid pupil recording" % new_rec_dir)
# parse info.csv file
meta_info_path = os.path.join(rec_dir, "info.csv")
with open(meta_info_path) as info:
meta_info = dict(((line.strip().split("\t")) for line in info.readlines()))
video_path = os.path.join(rec_dir, "world.mkv")
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
rec_version = read_rec_version(meta_info)
if rec_version >= VersionFormat("0.5"):
pass
elif rec_version >= VersionFormat("0.4"):
update_recording_0v4_to_current(rec_dir)
elif rec_version >= VersionFormat("0.3"):
update_recording_0v3_to_current(rec_dir)
video_path = os.path.join(rec_dir, "world.avi")
timestamps_path = os.path.join(rec_dir, "timestamps.npy")
else:
logger.Error("This recording is to old. Sorry.")
return
# Initialize capture
cap = autoCreateCapture(video_path, timestamps=timestamps_path)
if isinstance(cap, FakeCapture):
logger.error("could not start capture.")
return
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version", VersionFormat("0.0")) < get_version(version_file):
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
width, height = session_settings.get("window_size", cap.frame_size)
window_pos = session_settings.get("window_position", (0, 0))
main_window = glfwCreateWindow(
width,
height,
"Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1],
None,
None,
)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data["pupil_positions"]
gaze_list = pupil_data["gaze_positions"]
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = "player"
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = np.load(timestamps_path)
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug("Open Plugin: %s" % plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get("gui_scale", 1)
g_pool.main_menu = ui.Growing_Menu("Settings", pos=(-350, 20), size=(300, 400))
g_pool.main_menu.append(ui.Button("quit", lambda: on_close(None)))
g_pool.main_menu.append(
ui.Slider("scale", g_pool.gui, setter=set_scale, step=0.05, min=0.75, max=2.5, label="Interface Size")
)
g_pool.main_menu.append(ui.Info_Text("Player Version: %s" % g_pool.version))
g_pool.main_menu.append(ui.Info_Text("Recording Version: %s" % rec_version))
g_pool.main_menu.append(
ui.Selector(
"Open plugin",
selection=user_launchable_plugins,
labels=[p.__name__.replace("_", " ") for p in user_launchable_plugins],
setter=open_plugin,
getter=lambda: "Select to load",
)
)
g_pool.main_menu.append(ui.Button("Close all plugins", purge_plugins))
g_pool.main_menu.append(
ui.Button("Reset window size", lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1]))
)
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100))
g_pool.play_button = ui.Thumb("play", g_pool, label="Play", hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0, 1.0, 0.0, 0.8)
g_pool.forward_button = ui.Thumb("forward", getter=lambda: False, setter=next_frame, hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb("backward", getter=lambda: False, setter=prev_frame, hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [("Trim_Marks", {}), ("Seek_Bar", {})]
default_plugins = [("Scan_Path", {}), ("Vis_Polyline", {}), ("Vis_Circle", {}), ("Export_Launcher", {})]
previous_plugins = session_settings.get("loaded_plugins", default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins + previous_plugins)
for p in g_pool.plugins:
if p.class_name == "Trim_Marks":
g_pool.trim_marks = p
break
g_pool.gui.configuration = session_settings.get("ui_config", {})
# trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height, width, 3))
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now() - 0.03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
# grab new frame
if g_pool.play or g_pool.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
if g_pool.new_seek:
display_time = new_frame.timestamp
g_pool.new_seek = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
# new positons we make a deepcopy just like the image is a copy.
events["gaze_positions"] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events["pupil_positions"] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events["pupil_positions"]:
pupil_graph.add(p["confidence"])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1.0 / dt)
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex, frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
# present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time() - timestamp
sleep(wait_time - spent_time)
except:
pass
timestamp = time()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["gui_scale"] = g_pool.gui.scale
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["window_size"] = glfwGetWindowSize(main_window)
session_settings["window_position"] = glfwGetWindowPos(main_window)
session_settings["version"] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
g_pool.gui.terminate()
destroy_named_texture(g_pool.image_tex)
glfwDestroyWindow(main_window)
def eye(timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# general imports
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible
from ui_roi import UIRoi
# monitoring
import psutil
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, AV_Writer
from video_capture import source_classes
from video_capture import manager_classes
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D
}
# UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 10.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
# g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.process = 'eye{}'.format(eye_id)
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(
pos,
g_pool.capture.frame_size) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size + 40,
g_pool.u_r.handle_size + 40):
# if the roi interacts we dont want
# the gui to interact as well
return
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = glfw.glfwGetFramebufferSize(
window)[0] / glfw.glfwGetWindowSize(window)[0]
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, g_pool.capture.frame_size)
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir,
'user_settings_eye{}'.format(eye_id)))
if session_settings.get("version",
VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode',
'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."
}
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager', {}))
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__: c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](
g_pool, **manager_settings)
if eye_id == 0:
cap_src = [
"Pupil Cam1 ID0", "HD-6000", "Integrated Camera",
"HD USB Camera", "USB 2.0 Camera"
]
else:
cap_src = ["Pupil Cam1 ID1", "HD-6000", "Integrated Camera"]
# Initialize capture
default_settings = ('UVC_Source', {
'preferred_names': cap_src,
'frame_size': (640, 480),
'frame_rate': 90
})
capture_source_settings = overwrite_cap_settings or session_settings.get(
'capture_settings', default_settings)
source_class_name, source_settings = capture_source_settings
source_class_by_name = {c.__name__: c for c in source_classes}
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings)
assert g_pool.capture
g_pool.u_r = UIRoi(
(g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,
pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "Pupil Capture - eye {}".format(eye_id)
width, height = session_settings.get('window_size',
g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(
np.ones((1, 1), dtype=np.uint8) + 125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, *g_pool.capture.frame_size)))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector(
'pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Detector_2D, Detector_3D],
labels=['C++ 2d detector', 'C++ 3d detector'],
label="Detection method")
general_settings.append(detector_selector)
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture.init_gui()
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
g_pool.capture_manager.init_gui()
g_pool.writer = None
def replace_source(source_class_name, source_settings):
g_pool.capture.cleanup()
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings)
g_pool.capture.init_gui()
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
np.save(timestamps_path, np.asarray(timestamps))
g_pool.replace_source = replace_source # for ndsi capture
def replace_manager(manager_class):
g_pool.capture_manager.cleanup()
g_pool.capture_manager = manager_class(g_pool)
g_pool.capture_manager.init_gui()
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(
0,
ui.Selector('capture_manager',
g_pool,
setter=replace_manager,
getter=lambda: g_pool.capture_manager.__class__,
selection=manager_classes,
labels=[b.gui_name for b in manager_classes],
label='Manager'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
# set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
should_publish_frames = False
frame_publish_format = 'jpeg'
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification['subject']
if subject == 'eye_process.should_stop':
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye'] and g_pool.capture.online:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info(
"Will save eye video to: {}".format(record_path))
timestamps_path = os.path.join(
record_path, "eye{}_timestamps.npy".format(eye_id))
if raw_mode and frame and g_pool.capture.jpeg_support:
video_path = os.path.join(
record_path, "eye{}.mp4".format(eye_id))
writer = JPEG_Writer(video_path,
g_pool.capture.frame_rate)
else:
video_path = os.path.join(
record_path, "eye{}.mp4".format(eye_id))
writer = AV_Writer(video_path,
g_pool.capture.frame_rate)
timestamps = []
elif subject == 'recording.stopped':
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
np.save(timestamps_path, np.asarray(timestamps))
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject': 'meta.doc',
'actor': 'eye{}'.format(eye_id),
'doc': eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format', 'jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
elif subject.startswith('start_eye_capture') and notification[
'target'] == g_pool.process:
replace_source(notification['name'], notification['args'])
g_pool.capture.on_notify(notification)
# Get an image from the grabber
event = {}
g_pool.capture.recent_events(event)
frame = event.get('frame')
g_pool.capture_manager.recent_events(event)
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0],
g_pool.u_r.array_shape[1]) != (f_height, f_width):
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames and frame.jpeg_buffer:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
pupil_socket.send(
'frame.eye.%s' % eye_id, {
'width': frame.width,
'height': frame.width,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s' % eye_id, result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
if frame:
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
f_width, f_height = g_pool.capture.frame_size
make_coord_system_pixel_based((f_height, f_width, 3),
g_pool.flip)
if frame:
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly(
(int(eye_ball['center'][0]),
int(eye_ball['center'][1])),
(int(eye_ball['axes'][0] / 2),
int(eye_ball['axes'][1] / 2)),
int(eye_ball['angle']), 0, 360, 8)
except ValueError as e:
pass
else:
draw_polyline(
pts, 2,
RGBA(0., .9, .1,
result['model_confidence']))
if result['confidence'] > 0:
if 'ellipse' in result:
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360,
15)
confidence = result['confidence'] * 0.7
draw_polyline(pts, 1,
RGBA(1., 0, 0, confidence))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., confidence),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize(
) # detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings[
'capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict(
)
session_settings[
'capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict(
)
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = g_pool.version
session_settings[
'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings[
'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_gui()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
logger.info("Process shutting down.")
def session(rec_dir):
# Callback functions
def on_resize(window,w, h):
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*y_scroll_factor)
def on_drop(window,count,paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'"%new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window,True)
else:
logger.error("'%s' is not a valid pupil recording"%new_rec_dir)
tick = delta_t()
def get_dt():
return next(tick)
video_path = [f for f in glob(os.path.join(rec_dir,"world.*")) if f[-3:] in ('mp4','mkv','avi')][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
#parse info.csv file
meta_info_path = os.path.join(rec_dir,"info.csv")
with open(meta_info_path) as info:
meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) )
rec_version = read_rec_version(meta_info)
if rec_version >= VersionFormat('0.7.4'):
pass
if rec_version >= VersionFormat('0.7.3'):
update_recording_0v73_to_current(rec_dir)
elif rec_version >= VersionFormat('0.5'):
update_recording_0v5_to_current(rec_dir)
elif rec_version >= VersionFormat('0.4'):
update_recording_0v4_to_current(rec_dir)
elif rec_version >= VersionFormat('0.3'):
update_recording_0v3_to_current(rec_dir)
timestamps_path = os.path.join(rec_dir, "timestamps.npy")
else:
logger.Error("This recording is to old. Sorry.")
return
timestamps = np.load(timestamps_path)
# Initialize capture
cap = File_Capture(video_path,timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
if session_settings.get("version",VersionFormat('0.0')) < get_version(version_file):
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
width,height = session_settings.get('window_size',cap.frame_size)
window_pos = session_settings.get('window_position',(0,0))
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.binocular = meta_info.get('Eye Mode','monocular') == 'binocular'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.pupil_positions_by_frame = correlate_data(pupil_list,g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list,g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] #populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except FileSeekError:
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = slice(g_pool.trim_marks.in_mark,g_pool.trim_marks.out_mark)
export_dir = os.path.join(g_pool.rec_dir,'exports','%s-%s'%(export_range.start,export_range.stop))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
logger.warning("Previous export for range [%s-%s] already exsits - overwriting."%(export_range.start,export_range.stop))
else:
logger.info('Created export dir at "%s"'%export_dir)
notification = {'subject':'should_export','range':export_range,'export_dir':export_dir}
g_pool.notifications.append(notification)
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.main_menu = ui.Growing_Menu("Settings",pos=(-350,20),size=(300,400))
g_pool.main_menu.append(ui.Button("Close Pupil Player",lambda:glfwSetWindowShouldClose(main_window,True)))
g_pool.main_menu.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=0.75,max=2.5,label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version))
g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter = lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins))
g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.frame_size[0],cap.frame_size[1])) )
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0,1.,.0,.8)
g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT)
g_pool.export_button = ui.Thumb('export',getter = lambda: False, setter = do_export, hotkey='e')
g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button,g_pool.export_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})]
default_plugins = [('Log_Display',{}),('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Video_Export_Launcher',{})]
previous_plugins = session_settings.get('loaded_plugins',default_plugins)
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window,on_resize)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
glfwSetDropCallback(main_window,on_drop)
#trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config',{})
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_timestamp()-.03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play=False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt,ts = t-ts,t
fps_graph.add(1./dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
is_alive = Is_Alive_Manager(is_alive_flag)
with is_alive:
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level+10)
# create formatter and add it to the handlers
formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from av_writer import JPEG_Writer,AV_Writer
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':(640,480),'frame_rate':60}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
# pipe_to_world.send('eye%s process ready'%eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip',False)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
g_pool.u_r = UIRoi(frame.img.shape)
g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get()))
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1],new_size[0]))
cap.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s"%eye_id
width,height = session_settings.get('window_size',(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width,height, title, None, None)
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfw.glfwSwapInterval(0)
sphere = Sphere(20)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
general_settings.append(detector_selector)
# let detector add its GUI
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1/60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if pipe_to_world.poll():
cmd = pipe_to_world.recv()
if cmd == 'Exit':
break
elif cmd == "Ping":
pipe_to_world.send("Pong")
command = None
else:
command,payload = cmd
if command == 'Set_Detection_Mapping_Mode':
if payload == '3d':
if not isinstance(g_pool.pupil_detector,Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
set_detector(Detector_2D)
detector_selector.read_only = False
else:
command = None
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
cap.seek_to_frame(0)
frame = cap.get_frame()
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if 'Rec_Start' == command:
record_path,raw_mode = payload
logger.info("Will save eye video to: %s"%record_path)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = JPEG_Writer(video_path,cap.frame_rate)
else:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = AV_Writer(video_path,cap.frame_rate)
timestamps = []
elif 'Rec_Stop' == command:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
(int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
int(eye_ball['angle']),0,360,8)
except ValueError as e:
pass
else:
draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )
if result['confidence'] >0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
int(result['ellipse']['angle']),0,360,15)
confidence = result['confidence'] * 0.7 #scale it a little
draw_polyline(pts,1,RGBA(1.,0,0,confidence))
draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
g_pool.u_r.draw(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
logger.debug("Process done")
def main():
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor, h*hdpi_factor
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfwMakeContextCurrent(active_window)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*y_scroll_factor)
def on_close(window):
glfwSetWindowShouldClose(main_window,True)
logger.debug('Process closing from window')
try:
rec_dir = sys.argv[1]
except:
#for dev, supply hardcoded dir:
rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
if os.path.isdir(rec_dir):
logger.debug("Dev option: Using hadcoded data dir.")
else:
if getattr(sys, 'frozen', False):
logger.warning("You did not supply a data directory when you called this script! \
\nPlease drag a Pupil recoding directory onto the launch icon.")
else:
logger.warning("You did not supply a data directory when you called this script! \
\nPlease supply a Pupil recoding directory as first arg when calling Pupil Player.")
return
if not is_pupil_rec_dir(rec_dir):
logger.error("You did not supply a dir with the required files inside.")
return
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
#backwards compatibility fn.
patch_meta_info(rec_dir)
#parse info.csv file
meta_info_path = rec_dir + "/info.csv"
with open(meta_info_path) as info:
meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) )
rec_version = read_rec_version(meta_info)
if rec_version < VersionFormat('0.4'):
video_path = rec_dir + "/world.avi"
timestamps_path = rec_dir + "/timestamps.npy"
else:
video_path = rec_dir + "/world.mkv"
timestamps_path = rec_dir + "/world_timestamps.npy"
gaze_positions_path = rec_dir + "/gaze_positions.npy"
#load gaze information
gaze_list = np.load(gaze_positions_path)
timestamps = np.load(timestamps_path)
#correlate data
if rec_version < VersionFormat('0.4'):
positions_by_frame = correlate_gaze_legacy(gaze_list,timestamps)
else:
positions_by_frame = correlate_gaze(gaze_list,timestamps)
# Initialize capture
cap = autoCreateCapture(video_path,timestamps=timestamps_path)
if isinstance(cap,FakeCapture):
logger.error("could not start capture.")
return
width,height = session_settings.get('window_size',cap.get_size())
window_pos = session_settings.get('window_position',(0,0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# create container for globally scoped vars (within world)
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.gaze_list = gaze_list
g_pool.positions_by_frame = positions_by_frame
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s'%plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive=False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.main_menu = ui.Scrolling_Menu("Settings",pos=(-350,20),size=(300,300))
g_pool.main_menu.append(ui.Button("quit",lambda: on_close(None)))
g_pool.main_menu.configuration = session_settings.get('main_menu_config',{})
g_pool.main_menu.append(ui.Slider('scale', setter=set_scale,getter=get_scale,step = .05,min=0.75,max=2.5,label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s'%g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s'%rec_version))
g_pool.main_menu.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter = lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins',purge_plugins))
g_pool.main_menu.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,cap.get_size()[0],cap.get_size()[1])) )
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.play_button = ui.Thumb('play',g_pool,label='Play',hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0,1.,.0,.8)
g_pool.forward_button = ui.Thumb('forward',getter = lambda: False,setter= next_frame, hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',getter = lambda: False, setter = prev_frame, hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend([g_pool.play_button,g_pool.forward_button,g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks',{}),('Seek_Bar',{})]
default_plugins = [('Scan_Path',{}),('Vis_Polyline',{}),('Vis_Circle',{}),('Export_Launcher',{})]
previous_plugins = session_settings.get('loaded_plugins',default_plugins)
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,system_plugins+previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
glfwSetWindowPos(main_window,0,0)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height,width,3))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now()-.03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,110)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play=False
if g_pool.new_seek:
display_time = new_frame.timestamp
g_pool.new_seek = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#new positons we make a deepcopy just like the image is a copy.
events['pupil_positions'] = deepcopy(positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt,ts = t-ts,t
fps_graph.add(1./dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex,frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time()-timestamp
sleep(wait_time-spent_time)
except:
pass
timestamp = time()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['main_menu_config'] = g_pool.main_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def world(pupil_queue,timebase,launcher_pipe,eye_pipes,eyes_are_alive,user_dir,version,cap_src):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
# create formatter
formatter = logging.Formatter('%(processName)s - [%(levelname)s] %(name)s : %(message)s')
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir,'capture.log'),mode='w')
fh.setLevel(logger.level)
fh.setFormatter(formatter)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level+10)
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
#setup thread to recv log recrods from other processes.
def log_loop(logging):
import zmq
ctx = zmq.Context()
sub = ctx.socket(zmq.SUB)
sub.bind('tcp://127.0.0.1:502020')
sub.setsockopt(zmq.SUBSCRIBE, "")
while True:
record = sub.recv_pyobj()
logger = logging.getLogger(record.name)
logger.handle(record)
import threading
log_thread = threading.Thread(target=log_loop, args=(logging,))
log_thread.setDaemon(True)
log_thread.start()
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
#general imports
from time import time,sleep
import numpy as np
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen,make_coord_system_pixel_based,make_coord_system_norm_based
#check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
assert pyglui_version >= '0.8'
#monitoring
import psutil
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from version_utils import VersionFormat
import audio
# Plug-ins
from plugin import Plugin_List,import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from recorder import Recorder
from show_calibration import Show_Calibration
from display_recent_gaze import Display_Recent_Gaze
from pupil_server import Pupil_Server
from pupil_sync import Pupil_Sync
from surface_tracker import Surface_Tracker
from log_display import Log_Display
from annotations import Annotation_Capture
from pupil_remote import Pupil_Remote
from log_history import Log_History
from game_controller import GameController
logger.info('Application Version: %s'%version)
logger.info('System Info: %s'%get_system_info())
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (0,0)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (8,31)
else:
scroll_factor = 1.0
window_position_default = (0,0)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# g_pool.launcher_pipe = launcher_pipe
g_pool.eye_pipes = eye_pipes
g_pool.eyes_are_alive = eyes_are_alive
#manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
user_launchable_plugins = [GameController, Show_Calibration,Pupil_Remote,Pupil_Server,Pupil_Sync,Surface_Tracker,Annotation_Capture,Log_History]+runtime_plugins
system_plugins = [Log_Display,Display_Recent_Gaze,Recorder]
plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index,plugin_by_index))
default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':(1280,720),'frame_rate':30}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
launcher_pipe.send("Exit")
return
g_pool.iconified = False
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','2d')
g_pool.active_calibration_plugin = None
audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def launch_eye_process(eye_id,blocking=False):
if eyes_are_alive[eye_id].value:
logger.error("Eye%s process already running."%eye_id)
return
launcher_pipe.send(eye_id)
eye_pipes[eye_id].send( ('Set_Detection_Mapping_Mode',g_pool.detection_mapping_mode) )
if blocking:
#wait for ready message from eye to sequentialize startup
eye_pipes[eye_id].send('Ping')
eye_pipes[eye_id].recv()
logger.warning('Eye %s process started.'%eye_id)
def stop_eye_process(eye_id,blocking=False):
if eyes_are_alive[eye_id].value:
eye_pipes[eye_id].send('Exit')
if blocking:
while eyes_are_alive[eye_id].value:
sleep(.1)
def start_stop_eye(eye_id,make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
if new_mode == '2d':
for p in g_pool.plugins:
if "Vector_Gaze_Mapper" in p.class_name:
logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
p.alive = False
g_pool.plugins.clean()
for alive, pipe in zip(g_pool.eyes_are_alive,g_pool.eye_pipes):
if alive.value:
pipe.send( ('Set_Detection_Mapping_Mode',new_mode) )
g_pool.detection_mapping_mode = new_mode
#window and gl setup
glfw.glfwInit()
width,height = session_settings.get('window_size',(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width,height, "World")
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes))
general_settings.append(ui.Selector('detection_mapping_mode',g_pool,label='detection & mapping mode',setter=set_detection_mapping_mode,selection=['2d','3d']))
general_settings.append(ui.Switch('eye0_process',label='Detect eye 0',setter=lambda alive: start_stop_eye(0,alive),getter=lambda: eyes_are_alive[0].value ))
general_settings.append(ui.Switch('eye1_process',label='Detect eye 1',setter=lambda alive: start_stop_eye(1,alive),getter=lambda: eyes_are_alive[1].value ))
general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter=lambda: "Select to load"))
general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
g_pool.sidebar.append(general_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.gui.append(g_pool.quickbar)
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= open_plugin,label='Method'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
# refresh speed settings
glfw.glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
if session_settings.get('eye1_process_alive',False):
launch_eye_process(1,blocking=True)
if session_settings.get('eye0_process_alive',True):
launch_eye_process(0,blocking=False)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Starting Fake Capture.")
settings = g_pool.capture.settings
g_pool.capture.close()
g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase)
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.settings = settings
g_pool.notifications.append({'subject':'should_stop_recording'})
continue
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifications:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfw.glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
else:
g_pool.image_tex.update_from_frame(frame)
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture.close()
#shut down eye processes:
stop_eye_process(0,blocking = True)
stop_eye_process(1,blocking = True)
#shut down laucher
launcher_pipe.send("Exit")
logger.info("Process Shutting down.")
update_named_texture(g_pool.image_tex,frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
# fps_graph.draw()
# cpu_graph.draw()
# pupil_graph.draw()
# pupil_dia_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['side_bar_config'] = g_pool.sidebar.configuration
session_settings['capture_menu_config'] = g_pool.capture.menu.configuration
session_settings['general_menu_config'] = general_settings.configuration
session_settings['advanced_menu_config'] = advanced_settings.configuration
session_settings['calibration_menu_config']=g_pool.calibration_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
def session(rec_dir):
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__)
analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__)
other_plugins = sorted([Show_Calibration, Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__)
user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__)
user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
update_recording_to_recent(rec_dir)
video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
rec_version = read_rec_version(meta_info)
app_version = get_version(version_file)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
timestamps = np.load(timestamps_path)
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
# Initialize capture
cap = File_Source(g_pool, video_path, timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version", VersionFormat('0.0')) < get_version(version_file):
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', window_position_default)
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
+ rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except(FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except(FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if cap.get_frame_index() >= cap.get_frame_count()-5:
cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.delayed_notifications[notification['subject']] = notification
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark)
export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(export_range.start, export_range.stop))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.notifications.append(notification)
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500))
g_pool.main_menu.append(ui.Button("Close Pupil Player", lambda: glfwSetWindowShouldClose(main_window, True)))
g_pool.main_menu.append(ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=0.75, max=2.5, label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
g_pool.main_menu.append(ui.Info_Text('Recording Version: {}'.format(rec_version)))
g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
step=.05, min=0.0, max=1.0, label='Confidence threshold'))
selector_label = "Select to load"
vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins]
analysis_labels = [" " + p.__name__.replace('_', ' ') for p in analysis_plugins]
other_labels = [" " + p.__name__.replace('_', ' ') for p in other_plugins]
user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins]
plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label]
+ other_plugins + [selector_label] + user_plugins)
labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"]
+ other_labels + ["User added"] + user_labels)
g_pool.main_menu.append(ui.Selector('Open plugin:',
selection=plugins,
labels=labels,
setter=open_plugin,
getter=lambda: selector_label))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(ui.Button('Reset window size',
lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=chr(0xf04b),
setter=toggle_play,
hotkey=GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xf04e),
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xf04a),
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=chr(0xf063),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
# trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = None
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
# grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts and ts != t:
dt, ts = t-ts, t
fps_graph.add(1./dt)
else:
ts = new_frame.timestamp
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# publish delayed notifiactions when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
gl_utils.make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
gl_utils.make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
# present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.cleanup()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
def world(timebase,eyes_are_alive,ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version,cap_src):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``
``eye_process.started``
``start_plugin``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
#general imports
from time import time,sleep
import numpy as np
import logging
import zmq
import zmq_tools
#zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url)
gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url)
pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('pupil',))
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('notify',))
#log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen,make_coord_system_pixel_based,make_coord_system_norm_based,glFlush
#check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
assert pyglui_version >= '0.8'
#monitoring
import psutil
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from version_utils import VersionFormat
import audio
from uvc import get_time_monotonic
#trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin,Plugin_List,import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from recorder import Recorder
from show_calibration import Show_Calibration
from display_recent_gaze import Display_Recent_Gaze
from pupil_sync import Pupil_Sync
from pupil_remote import Pupil_Remote
from surface_tracker import Surface_Tracker
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
logger.info('Application Version: %s'%version)
logger.info('System Info: %s'%get_system_info())
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (0,0)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (8,31)
else:
scroll_factor = 1.0
window_position_default = (0,0)
#g_pool holds variables for this process they are accesible to all plugins
g_pool = Global_Container()
g_pool.app = 'capture'
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eyes_are_alive = eyes_are_alive
def get_timestamp():
return get_time_monotonic()-g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
#manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
user_launchable_plugins = [Show_Calibration,Pupil_Remote,Pupil_Sync,Surface_Tracker,Annotation_Capture,Log_History]+runtime_plugins
system_plugins = [Log_Display,Display_Recent_Gaze,Recorder]
plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index,plugin_by_index))
default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{}),('Pupil_Remote',{})]
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':(1280,720),'frame_rate':30}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
g_pool.iconified = False
g_pool.capture = cap
g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','2d')
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def launch_eye_process(eye_id,delay=0):
n = {'subject':'eye_process.should_start.%s'%eye_id,'eye_id':eye_id,'delay':delay}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {'subject':'eye_process.should_stop','eye_id':eye_id}
ipc_pub.notify(n)
def start_stop_eye(eye_id,make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
n = {'subject':'set_detection_mapping_mode','mode':new_mode}
ipc_pub.notify(n)
def handle_notifications(n):
subject = n['subject']
if subject == 'set_detection_mapping_mode':
if n['mode'] == '2d':
if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name:
logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
g_pool.detection_mapping_mode = n['mode']
elif subject == 'start_plugin':
g_pool.plugins.add(plugin_by_name[n['name']],args=n.get('args',{}) )
elif subject == 'eye_process.started':
n = {'subject':'set_detection_mapping_mode','mode':g_pool.detection_mapping_mode}
ipc_pub.notify(n)
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject':'meta.doc',
'actor':g_pool.app,
'doc':world.__doc__})
for p in g_pool.plugins:
if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify:
ipc_pub.notify({
'subject':'meta.doc',
'actor': p.class_name,
'doc':p.on_notify.__doc__})
#window and gl setup
glfw.glfwInit()
width,height = session_settings.get('window_size',cap.frame_size)
main_window = glfw.glfwCreateWindow(width,height, "World")
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes))
general_settings.append(ui.Selector('detection_mapping_mode',g_pool,label='detection & mapping mode',setter=set_detection_mapping_mode,selection=['2d','3d']))
general_settings.append(ui.Switch('eye0_process',label='Detect eye 0',setter=lambda alive: start_stop_eye(0,alive),getter=lambda: eyes_are_alive[0].value ))
general_settings.append(ui.Switch('eye1_process',label='Detect eye 1',setter=lambda alive: start_stop_eye(1,alive),getter=lambda: eyes_are_alive[1].value ))
general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter=lambda: "Select to load"))
general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
g_pool.sidebar.append(general_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.gui.append(g_pool.quickbar)
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= open_plugin,label='Method'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
# refresh speed settings
glfw.glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil0_graph = graph.Bar_Graph(max_val=1.0)
pupil0_graph.pos = (260,130)
pupil0_graph.update_rate = 5
pupil0_graph.label = "id0 conf: %0.2f"
pupil1_graph = graph.Bar_Graph(max_val=1.0)
pupil1_graph.pos = (380,130)
pupil1_graph.update_rate = 5
pupil1_graph.label = "id1 conf: %0.2f"
pupil_graphs = pupil0_graph,pupil1_graph
if session_settings.get('eye1_process_alive',False):
launch_eye_process(1,delay=0.6)
if session_settings.get('eye0_process_alive',True):
launch_eye_process(0,delay=0.3)
ipc_pub.notify({'subject':'world_process.started'})
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Starting Fake Capture.")
settings = g_pool.capture.settings
g_pool.capture.close()
g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase)
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.settings = settings
ipc_pub.notify({'subject':'recording.should_stop'})
continue
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
recent_pupil_data = []
recent_gaze_data = []
new_notifications = []
while pupil_sub.new_data:
t,p = pupil_sub.recv()
pupil_graphs[p['id']].add(p['confidence'])
recent_pupil_data.append(p)
new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p)
for g in new_gaze_data:
gaze_pub.send('gaze',g)
recent_gaze_data += new_gaze_data
while notify_sub.new_data:
t,n = notify_sub.recv()
new_notifications.append(n)
events['pupil_positions'] = recent_pupil_data
events['gaze_positions'] = recent_gaze_data
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
#send new events to ipc:
del events['pupil_positions'] #already on the wire
del events['gaze_positions'] #send earlier in this loop
del events['dt'] #no need to send this
for topic,data in events.iteritems():
for d in data:
ipc_pub.send(topic, d)
# render camera image
glfw.glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
else:
g_pool.image_tex.update_from_frame(frame)
glFlush()
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil0_graph.draw()
pupil1_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture.close()
#shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({'subject':'world_process.stopped'})
#shut down launcher
n = {'subject':'launcher_process.should_stop'}
ipc_pub.notify(n)
zmq_ctx.destroy()
def main():
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(
window)[0]
w, h = w * hdpi_factor, h * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfwMakeContextCurrent(active_window)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_close(window):
glfwSetWindowShouldClose(main_window, True)
logger.debug('Process closing from window')
try:
rec_dir = sys.argv[1]
except:
#for dev, supply hardcoded dir:
rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
if os.path.isdir(rec_dir):
logger.debug("Dev option: Using hadcoded data dir.")
else:
if getattr(sys, 'frozen', False):
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease drag a Pupil recoding directory onto the launch icon."
)
else:
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease supply a Pupil recoding directory as first arg when calling Pupil Player."
)
return
if not is_pupil_rec_dir(rec_dir):
logger.error(
"You did not supply a dir with the required files inside.")
return
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
#backwards compatibility fn.
patch_meta_info(rec_dir)
#parse info.csv file
meta_info_path = rec_dir + "/info.csv"
with open(meta_info_path) as info:
meta_info = dict(
((line.strip().split('\t')) for line in info.readlines()))
rec_version = read_rec_version(meta_info)
if rec_version < VersionFormat('0.4'):
video_path = rec_dir + "/world.avi"
timestamps_path = rec_dir + "/timestamps.npy"
else:
video_path = rec_dir + "/world.mkv"
timestamps_path = rec_dir + "/world_timestamps.npy"
gaze_positions_path = rec_dir + "/gaze_positions.npy"
#load gaze information
gaze_list = np.load(gaze_positions_path)
timestamps = np.load(timestamps_path)
#correlate data
if rec_version < VersionFormat('0.4'):
positions_by_frame = correlate_gaze_legacy(gaze_list, timestamps)
else:
positions_by_frame = correlate_gaze(gaze_list, timestamps)
# Initialize capture
cap = autoCreateCapture(video_path, timestamps=timestamps_path)
if isinstance(cap, FakeCapture):
logger.error("could not start capture.")
return
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position',
(0, 0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# create container for globally scoped vars (within world)
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.gaze_list = gaze_list
g_pool.positions_by_frame = positions_by_frame
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s' % plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Scrolling_Menu("Settings",
pos=(-350, 20),
size=(300, 300))
g_pool.main_menu.append(ui.Button("quit", lambda: on_close(None)))
g_pool.main_menu.configuration = session_settings.get(
'main_menu_config', {})
g_pool.main_menu.append(
ui.Slider('scale',
setter=set_scale,
getter=get_scale,
step=.05,
min=0.75,
max=2.5,
label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
rec_version))
g_pool.main_menu.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label='Play',
hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend(
[g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Scan_Path', {}), ('Vis_Polyline', {}),
('Vis_Circle', {}), ('Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
glfwSetWindowPos(main_window, 0, 0)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height, width, 3))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now() - .03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play = False
if g_pool.new_seek:
display_time = new_frame.timestamp
g_pool.new_seek = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#new positons we make a deepcopy just like the image is a copy.
events['pupil_positions'] = deepcopy(positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex, frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time() - timestamp
sleep(wait_time - spent_time)
except:
pass
timestamp = time()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['main_menu_config'] = g_pool.main_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def world(g_pool,cap_src,cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor, h*hdpi_factor
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
glfwMakeContextCurrent(active_window)
def on_iconify(window,iconfied):
pass
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = session_settings.get("update_textures",2)
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
g_pool.active_calibration_plugin = None
#UI callback functions
def reset_timebase():
#the last frame from worldcam will be t0
g_pool.timebase.value = g_pool.capture.get_now()
logger.info("New timebase set to %s all timestamps will count from here now."%g_pool.timebase.value)
def set_calibration_plugin(new_calibration):
g_pool.active_calibration_plugin = new_calibration
new_plugin = new_calibration(g_pool)
g_pool.plugins.add(new_plugin)
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s'%plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
width,height = session_settings.get('window_size',(frame.width, frame.height))
window_pos = session_settings.get('window_position',(0,0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(width,height, "World", None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetWindowIconifyCallback(main_window,on_iconify)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex,frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window,0,0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-250,0),size=(0,0),header_pos='left')
g_pool.sidebar.configuration = session_settings.get('side_bar_config',{})
general_settings = ui.Growing_Menu('General')
general_settings.configuration = session_settings.get('general_menu_config',{})
general_settings.append(ui.Slider('scale', setter=set_scale,getter=get_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.configuration = session_settings.get('advanced_menu_config',{'collapsed':True})
advanced_settings.append(ui.Selector('update_textures',g_pool,label="Update display",selection=range(3),labels=('No update','Gray','Color')))
advanced_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
advanced_settings.append(ui.Button('Set timebase to 0',reset_timebase))
advanced_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.configuration = session_settings.get('calibration_menu_config',{})
g_pool.calibration_menu.append(ui.Selector('active_calibration_plugin',g_pool, selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= set_calibration_plugin,label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.menu.configuration = session_settings.get('capture_menu_config',{})
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
#only needed for the gui to show the loaded calibration type
for p in g_pool.plugins:
if p.base_class_name == 'Calibration_Plugin':
g_pool.active_calibration_plugin = p.__class__
break
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex,frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['side_bar_config'] = g_pool.sidebar.configuration
session_settings['capture_menu_config'] = g_pool.capture.menu.configuration
session_settings['general_menu_config'] = general_settings.configuration
session_settings['advanced_menu_config'] = advanced_settings.configuration
session_settings['calibration_menu_config']=g_pool.calibration_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['update_textures'] = g_pool.update_textures
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def eye(g_pool,cap_src,cap_size,rx_from_world,eye_id=0):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye%s.log'%eye_id),mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level+10)
# create formatter and add it to the handlers
formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor, h*hdpi_factor
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
# for p in g_pool.plugins:
# p.on_window_resize(window,w,h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == GLFW_RELEASE and u_r.active_edit_pt:
u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if u_r.mouse_over_edit_pt(pos,u_r.handle_size+40,u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if u_r.active_edit_pt:
pos = normalize((x,y),glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
u_r.move_vertex(u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
cap.frame_size = cap_size
cap.frame_rate = 90 #default
cap.settings = session_settings.get('capture_settings',{})
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
g_pool.capture = cap
g_pool.flip = session_settings.get('flip',False)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.window_size = session_settings.get('window_size',1.)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."}
# g_pool.draw_pupil = session_settings.get('draw_pupil',True)
u_r = UIRoi(frame.img.shape)
u_r.set(session_settings.get('roi',u_r.get()))
writer = None
pupil_detector = Canny_Detector(g_pool)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
window_pos = session_settings.get('window_position',window_position_default)
width,height = session_settings.get('window_size',(frame.width, frame.height))
# Initialize glfw
glfwInit()
if g_pool.binocular:
title = "Binocular eye %s"%eye_id
else:
title = 'Eye'
main_window = glfwCreateWindow(width,height, title, None, None)
glfwMakeContextCurrent(main_window)
cygl_init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex,frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# let detector add its GUI
pupil_detector.init_gui(g_pool.sidebar)
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if rx_from_world.poll():
command,raw_mode = rx_from_world.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s"%record_path)
video_path = os.path.join(record_path, "eye%s.mkv"%eye_id)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode:
writer = JPEG_Dumper(video_path)
else:
writer = CV_Writer(video_path,float(cap.frame_rate), cap.frame_size)
timestamps = []
else:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(frame,user_roi=u_r,visualize=g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
update_named_texture(g_pool.image_tex,frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
draw_named_texture(g_pool.image_tex)
# switch to work in pixel space
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
if result['confidence'] >0:
if result.has_key('axes'):
pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
(int(result['axes'][0]/2),int(result['axes'][1]/2)),
int(result['angle']),0,360,15)
cygl_draw_polyline(pts,1,cygl_rgba(1.,0,0,.5))
cygl_draw_points([result['center']],size=20,color=cygl_rgba(1.,0.,0.,.5),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
u_r.draw(g_pool.gui.scale)
#update screen
glfwSwapBuffers(main_window)
glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
pupil_detector.cleanup()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
#flushing queue in case world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def session(rec_dir):
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'" % new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'%s' is not a valid pupil recording" %
new_rec_dir)
tick = delta_t()
def get_dt():
return next(tick)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if f[-3:] in ('mp4', 'mkv', 'avi')
][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
#parse info.csv file
meta_info_path = os.path.join(rec_dir, "info.csv")
with open(meta_info_path) as info:
meta_info = dict(
((line.strip().split('\t')) for line in info.readlines()))
rec_version = read_rec_version(meta_info)
if rec_version >= VersionFormat('0.5'):
pass
elif rec_version >= VersionFormat('0.4'):
update_recording_0v4_to_current(rec_dir)
elif rec_version >= VersionFormat('0.3'):
update_recording_0v3_to_current(rec_dir)
timestamps_path = os.path.join(rec_dir, "timestamps.npy")
else:
logger.Error("This recording is to old. Sorry.")
return
# Initialize capture
cap = File_Capture(video_path, timestamps=timestamps_path)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version",
VersionFormat('0.0')) < get_version(version_file):
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', (0, 0))
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = np.load(timestamps_path)
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
] #populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Growing_Menu("Settings",
pos=(-350, 20),
size=(300, 400))
g_pool.main_menu.append(
ui.Button("Close Pupil Player",
lambda: glfwSetWindowShouldClose(main_window, True)))
g_pool.main_menu.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=0.75,
max=2.5,
label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
rec_version))
g_pool.main_menu.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label='Play',
hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend(
[g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Scan_Path', {}),
('Vis_Polyline', {}), ('Vis_Circle', {}),
('Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.notifications = []
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
#trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_timestamp() - .03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(
g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(
g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
def world(pupil_queue, timebase, lauchner_pipe, eye_pipes, eyes_are_alive,
user_dir, version, cap_src):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir, 'world.log'), mode='w')
fh.setLevel(logger.level)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'World Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'WORLD Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logging.getLogger("libav").setLevel(logging.ERROR)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmfull but unnessasary.
#general imports
from time import time
import numpy as np
#display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based
#check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
assert pyglui_version >= '0.8'
#monitoring
import psutil
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from version_utils import VersionFormat
import audio
# Plug-ins
from plugin import Plugin_List, import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from recorder import Recorder
from show_calibration import Show_Calibration
from display_recent_gaze import Display_Recent_Gaze
from pupil_server import Pupil_Server
from pupil_sync import Pupil_Sync
from marker_detector import Marker_Detector
from log_display import Log_Display
from annotations import Annotation_Capture
# create logger for the context of this function
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (0, 0)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (8, 31)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# g_pool.lauchner_pipe = lauchner_pipe
g_pool.eye_pipes = eye_pipes
g_pool.eyes_are_alive = eyes_are_alive
#manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, 'plugins'))
user_launchable_plugins = [
Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector,
Annotation_Capture
] + runtime_plugins
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
('Display_Recent_Gaze', {}),
('Screen_Marker_Calibration', {}), ('Recorder', {})]
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': (1280, 720), 'frame_rate': 30}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
lauchner_pipe.send("Exit")
return
g_pool.iconified = False
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get(
'pupil_confidence_threshold', .6)
g_pool.detection_mapping_mode = session_settings.get(
'detection_mapping_mode', '2d')
g_pool.active_calibration_plugin = None
audio.audio_mode = session_settings.get('audio_mode',
audio.default_audio_mode)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def launch_eye_process(eye_id, blocking=False):
if eyes_are_alive[eye_id].value:
logger.error("Eye%s process already running." % eye_id)
return
lauchner_pipe.send(eye_id)
eye_pipes[eye_id].send(
('Set_Detection_Mapping_Mode', g_pool.detection_mapping_mode))
if blocking:
#wait for ready message from eye to sequentialize startup
eye_pipes[eye_id].send('Ping')
eye_pipes[eye_id].recv()
logger.warning('Eye %s process started.' % eye_id)
def stop_eye_process(eye_id, blocking=False):
if eyes_are_alive[eye_id].value:
eye_pipes[eye_id].send('Exit')
if blocking:
raise NotImplementedError()
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
if new_mode == '2d':
for p in g_pool.plugins:
if "Vector_Gaze_Mapper" in p.class_name:
logger.warning(
"The gaze mapper is not supported in 2d mode. Please recalibrate."
)
p.alive = False
g_pool.plugins.clean()
for alive, pipe in zip(g_pool.eyes_are_alive, g_pool.eye_pipes):
if alive.value:
pipe.send(('Set_Detection_Mapping_Mode', new_mode))
g_pool.detection_mapping_mode = new_mode
#window and gl setup
glfw.glfwInit()
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width, height, "World")
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-350, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('audio_mode', audio, selection=audio.audio_modes))
general_settings.append(
ui.Selector('detection_mapping_mode',
g_pool,
label='detection & mapping mode',
setter=set_detection_mapping_mode,
selection=['2d', '3d']))
general_settings.append(
ui.Switch('eye0_process',
label='Detect eye 0',
setter=lambda alive: start_stop_eye(0, alive),
getter=lambda: eyes_are_alive[0].value))
general_settings.append(
ui.Switch('eye1_process',
label='Detect eye 1',
setter=lambda alive: start_stop_eye(1, alive),
getter=lambda: eyes_are_alive[1].value))
general_settings.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
general_settings.append(
ui.Slider('pupil_confidence_threshold',
g_pool,
step=.01,
min=0.,
max=1.,
label='Minimum pupil confidence'))
general_settings.append(
ui.Info_Text('Capture Version: %s' % g_pool.version))
g_pool.sidebar.append(general_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.gui.append(g_pool.quickbar)
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(
0,
ui.Selector(
'active_calibration_plugin',
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection=calibration_plugins,
labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
setter=open_plugin,
label='Method'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
# refresh speed settings
glfw.glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
if session_settings.get('eye1_process_alive', False):
launch_eye_process(1, blocking=True)
if session_settings.get('eye0_process_alive', True):
launch_eye_process(0, blocking=False)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifications:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfw.glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
else:
g_pool.image_tex.update_from_frame(frame)
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings[
'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
#shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
#shut down laucher
lauchner_pipe.send("Exit")
logger.debug("world process done")
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url,ipc_push_url, user_dir, version, eye_id,overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=("notify",))
with Is_Alive_Manager(is_alive_flag,ipc_socket,eye_id):
#logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based,is_window_visible
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from uvc import get_time_monotonic, StreamError
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from av_writer import JPEG_Writer,AV_Writer
from video_capture import InitialisationError,StreamError, Fake_Source,EndofVideoFileError, source_classes, manager_classes
source_by_name = {src.class_name():src for src in source_classes}
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic()-g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window,w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager',{}))
if eye_id == 0:
cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
else:
cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]
# Initialize capture
default_settings = {
'source_class_name': 'UVC_Source',
'preferred_names' : cap_src,
'frame_size': (640,480),
'frame_rate': 90
}
settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
try:
cap = source_by_name[settings['source_class_name']](g_pool, **settings)
except (KeyError,InitialisationError) as e:
if isinstance(e,KeyError):
logger.warning('Incompatible capture setting encountered. Falling back to fake source.')
cap = Fake_Source(g_pool, **settings)
g_pool.iconified = False
g_pool.capture = cap
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip',False)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
writer = None
pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s"%eye_id
width,height = session_settings.get('window_size',g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width,height, title, None, None)
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
glfw.glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
general_settings.append(detector_selector)
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture.init_gui()
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__:c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
g_pool.capture_manager.init_gui()
def open_manager(manager_class):
g_pool.capture_manager.cleanup()
g_pool.capture_manager = manager_class(g_pool)
g_pool.capture_manager.init_gui()
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(0,ui.Selector(
'capture_manager',g_pool,
setter = open_manager,
getter = lambda: g_pool.capture_manager.__class__,
selection = manager_classes,
labels = [b.gui_name for b in manager_classes],
label = 'Manager'
))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
should_publish_frames = False
frame_publish_format = 'jpeg'
#create a timer to control window update frequency
window_update_timer = timer(1/60.)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t,notification = notify_sub.recv()
subject = notification['subject']
if subject == 'eye_process.should_stop':
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector,Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector,Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye']:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: %s"%record_path)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = JPEG_Writer(video_path,g_pool.capture.frame_rate)
else:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = AV_Writer(video_path,g_pool.capture.frame_rate)
timestamps = []
elif subject == 'recording.stopped':
if writer:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject':'meta.doc',
'actor':'eye%i'%eye_id,
'doc':eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format','jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
else:
g_pool.capture_manager.on_notify(notification)
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except StreamError as e:
logger.error("Error getting frame. Stopping eye process.")
logger.debug("Caught error: %s"%e)
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
g_pool.capture.seek_to_frame(0)
frame = g_pool.capture.get_frame()
g_pool.u_r = UIRoi((frame.height,frame.width))
g_pool.capture_manager.update(frame, {})
if should_publish_frames and frame.jpeg_buffer:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
pupil_socket.send('frame.eye.%s'%eye_id,{
'width': frame.width,
'height': frame.width,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s'%eye_id,result)
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
g_pool.capture.gl_display()
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
(int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
int(eye_ball['angle']),0,360,8)
except ValueError as e:
pass
else:
draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )
if result['confidence'] >0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
int(result['ellipse']['angle']),0,360,15)
confidence = result['confidence'] * 0.7 #scale it a little
draw_polyline(pts,1,RGBA(1.,0,0,confidence))
draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_gui()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
logger.info("Process shutting down.")
def world(g_pool,cap_src,cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
#manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins
system_plugins = [Log_Display,Display_Recent_Gaze,Recorder]
plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index,plugin_by_index))
default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':cap_size,'frame_rate':24}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = session_settings.get("update_textures",2)
g_pool.iconified = False
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
g_pool.active_calibration_plugin = None
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
#window and gl setup
glfwInit()
width,height = session_settings.get('window_size',(frame.width, frame.height))
main_window = glfwCreateWindow(width,height, "World")
window_pos = session_settings.get('window_position',window_position_default)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.append(ui.Selector('update_textures',g_pool,label="Update display",selection=range(3),labels=('No update','Gray','Color')))
advanced_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
advanced_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.append(ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= open_plugin,label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(ui.Hot_Key("quit",setter=on_close,getter=lambda:True,label="X",hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetWindowIconifyCallback(main_window,on_iconify)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex,frame.img)
# refresh speed settings
glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
elif g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex,frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex,frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['update_textures'] = g_pool.update_textures
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
logger.debug("Process done")
def world(g_pool,cap_src,cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
#manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Sync,Marker_Detector]+runtime_plugins
system_plugins = [Log_Display,Display_Recent_Gaze,Recorder]
plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index,plugin_by_index))
default_plugins = [('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{})]
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':cap_size,'frame_rate':24}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6)
g_pool.active_calibration_plugin = None
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
#window and gl setup
glfwInit()
width,height = session_settings.get('window_size',(frame.width, frame.height))
main_window = glfwCreateWindow(width,height, "World")
window_pos = session_settings.get('window_position',window_position_default)
glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('Open plugin', selection = user_launchable_plugins,
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins],
setter= open_plugin, getter=lambda: "Select to load"))
general_settings.append(ui.Slider('pupil_confidence_threshold', g_pool,step = .01,min=0.,max=1.,label='Minimum pupil confidence'))
general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
g_pool.sidebar.append(general_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.gui.append(g_pool.quickbar)
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(0,ui.Selector('active_calibration_plugin',getter=lambda: g_pool.active_calibration_plugin.__class__, selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= open_plugin,label='Method'))
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetWindowIconifyCallback(main_window,on_iconify)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
# refresh speed settings
glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260,130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifications:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
else:
g_pool.image_tex.update_from_frame(frame)
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
logger.debug("Process done")
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version,
eye_id, cap_src):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
is_alive = Is_Alive_Manager(is_alive_flag)
with is_alive:
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir, 'eye%s.log' % eye_id),
mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'Eye' + str(eye_id) +
' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'EYE' + str(eye_id) +
' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logging.getLogger("libav").setLevel(logging.ERROR)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture
from OpenGL.GL import GL_LINE_LOOP
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based
from ui_roi import UIRoi
#monitoring
import psutil
# helpers/utils
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from av_writer import JPEG_Writer, AV_Writer
# Pupil detectors
from pupil_detectors import Canny_Detector, Detector_2D, Detector_3D
pupil_detectors = {
Canny_Detector.__name__: Canny_Detector,
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D
}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos,
(frame.width, frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size + 40,
g_pool.u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version",
VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
# pipe_to_world.send('eye%s process ready'%eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode',
'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
}
g_pool.u_r = UIRoi(frame.img.shape)
g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1], new_size[0]))
cap.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,
pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s" % eye_id
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfw.glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector(
'pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Canny_Detector, Detector_2D, Detector_3D],
labels=[
'Python 2D detector', 'C++ 2d detector', 'C++ 3d detector'
],
label="Detection method")
general_settings.append(detector_selector)
# let detector add its GUI
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1 / 60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if pipe_to_world.poll():
cmd = pipe_to_world.recv()
if cmd == 'Exit':
break
elif cmd == "Ping":
pipe_to_world.send("Pong")
command = None
else:
command, payload = cmd
if command == 'Set_Detection_Mapping_Mode':
if payload == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
set_detector(Detector_2D)
detector_selector.read_only = False
else:
command = None
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
cap.seek_to_frame(0)
frame = cap.get_frame()
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if 'Rec_Start' == command:
record_path, raw_mode = payload
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(record_path,
"eye%s_timestamps.npy" % eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path, cap.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = AV_Writer(video_path, cap.frame_rate)
timestamps = []
elif 'Rec_Stop' == command:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
# switch to work in pixel space
make_coord_system_pixel_based(
(frame.height, frame.width, 3), g_pool.flip)
if result['confidence'] > 0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
draw_polyline(pts, 1, RGBA(1., 0, 0, .5))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., .5),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
g_pool.u_r.draw(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize(
) #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = g_pool.version
session_settings[
'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings[
'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
logger.debug("Process done")
def world(g_pool, cap_src, cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(
window)[0]
w, h = w * hdpi_factor, h * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
glfwMakeContextCurrent(active_window)
def on_iconify(window, iconfied):
if not isinstance(cap, FakeCapture):
g_pool.update_textures = not iconfied
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = 2
if isinstance(cap, FakeCapture):
g_pool.update_textures = 0
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get(
'pupil_confidence_threshold', .6)
g_pool.active_calibration_plugin = None
#UI callback functions
def reset_timebase():
#the last frame from worldcam will be t0
g_pool.timebase.value = g_pool.capture.get_now()
logger.info(
"New timebase set to %s all timestamps will count from here now." %
g_pool.timebase.value)
def set_calibration_plugin(new_calibration):
g_pool.active_calibration_plugin = new_calibration
new_plugin = new_calibration(g_pool)
g_pool.plugins.add(new_plugin)
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s' % plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
width, height = session_settings.get('window_size',
(frame.width, frame.height))
window_pos = session_settings.get('window_position',
(0, 0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(width, height, "World", None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetWindowIconifyCallback(main_window, on_iconify)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex, frame.img)
# refresh speed settings
glfwSwapInterval(0)
glfwSetWindowPos(main_window, 0, 0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-250, 0),
size=(0, 0),
header_pos='left')
g_pool.sidebar.configuration = session_settings.get('side_bar_config', {})
general_settings = ui.Growing_Menu('General')
general_settings.configuration = session_settings.get(
'general_menu_config', {})
general_settings.append(
ui.Slider('scale',
setter=set_scale,
getter=get_scale,
step=.05,
min=1.,
max=2.5,
label='Interface size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.configuration = session_settings.get(
'advanced_menu_config', {'collapsed': True})
advanced_settings.append(
ui.Selector('update_textures',
g_pool,
label="Update display",
selection=range(3),
labels=('No update', 'Gray', 'Color')))
advanced_settings.append(
ui.Slider('pupil_confidence_threshold',
g_pool,
step=.01,
min=0.,
max=1.,
label='Minimum pupil confidence'))
advanced_settings.append(ui.Button('Set timebase to 0', reset_timebase))
advanced_settings.append(
ui.Info_Text('Capture Version: %s' % g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.configuration = session_settings.get(
'calibration_menu_config', {})
g_pool.calibration_menu.append(
ui.Selector(
'active_calibration_plugin',
g_pool,
selection=calibration_plugins,
labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
setter=set_calibration_plugin,
label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(
ui.Hot_Key("quit",
setter=on_close,
getter=lambda: True,
label="X",
hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.menu.configuration = session_settings.get(
'capture_menu_config', {})
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
#only needed for the gui to show the loaded calibration type
for p in g_pool.plugins:
if p.base_class_name == 'Calibration_Plugin':
g_pool.active_calibration_plugin = p.__class__
break
on_resize(main_window, *glfwGetWindowSize(main_window))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
fps_graph.add(1. / dt)
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex, frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex, frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings[
'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['side_bar_config'] = g_pool.sidebar.configuration
session_settings['capture_menu_config'] = g_pool.capture.menu.configuration
session_settings['general_menu_config'] = general_settings.configuration
session_settings['advanced_menu_config'] = advanced_settings.configuration
session_settings[
'calibration_menu_config'] = g_pool.calibration_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, version):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``
``eye_process.started``
``start_plugin``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
#general imports
from time import time, sleep
import numpy as np
import logging
#networking
import zmq
import zmq_tools
#zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('pupil', ))
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('notify', ))
#log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#display
import glfw
from pyglui import ui, graph, cygl, __version__ as pyglui_version
assert pyglui_version >= '1.0'
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush, is_window_visible
#monitoring
import psutil
# helpers/utils
from version_utils import VersionFormat
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info
from uvc import get_time_monotonic
logger.info('Application Version: %s' % version)
logger.info('System Info: %s' % get_system_info())
# video sources
from video_capture import InitialisationError, StreamError, Fake_Source, EndofVideoFileError, source_classes, manager_classes
source_by_name = {src.class_name(): src for src in source_classes}
import audio
#trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from fixation_detector import Fixation_Detector_3D
from recorder import Recorder
from show_calibration import Show_Calibration
from display_recent_gaze import Display_Recent_Gaze
from time_sync import Time_Sync
from pupil_remote import Pupil_Remote
from pupil_groups import Pupil_Groups
from surface_tracker import Surface_Tracker
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
from frame_publisher import Frame_Publisher
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (0, 0)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (8, 31)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
#g_pool holds variables for this process they are accesible to all plugins
g_pool = Global_Container()
g_pool.app = 'capture'
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eyes_are_alive = eyes_are_alive
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
#manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, 'plugins'))
user_launchable_plugins = [
Pupil_Groups, Frame_Publisher, Show_Calibration, Pupil_Remote,
Time_Sync, Surface_Tracker, Annotation_Capture, Log_History,
Fixation_Detector_3D
] + runtime_plugins
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins + manager_classes
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [('UVC_Manager', {}), ('Log_Display', {}),
('Dummy_Gaze_Mapper', {}), ('Display_Recent_Gaze', {}),
('Screen_Marker_Calibration', {}), ('Recorder', {}),
('Pupil_Remote', {}), ('Fixation_Detector_3D', {})]
# Callback functions
def on_resize(window, w, h):
if is_window_visible(window):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
tick = delta_t()
def get_dt():
return next(tick)
g_pool.on_frame_size_change = lambda new_size: None
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
default_settings = {
'source_class_name':
'UVC_Source',
'preferred_names': [
"Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525",
"C525", "C615", "C920", "C930e"
],
'frame_size': (1280, 720),
'frame_rate':
30
}
settings = session_settings.get('capture_settings', default_settings)
try:
cap = source_by_name[settings['source_class_name']](g_pool, **settings)
except (KeyError, InitialisationError) as e:
if isinstance(e, KeyError):
logger.warning(
'Incompatible capture setting encountered. Falling back to fake source.'
)
cap = Fake_Source(g_pool, **settings)
g_pool.iconified = False
g_pool.detection_mapping_mode = session_settings.get(
'detection_mapping_mode', '3d')
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
g_pool.capture = cap
g_pool.capture_manager = None
audio.audio_mode = session_settings.get('audio_mode',
audio.default_audio_mode)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def launch_eye_process(eye_id, delay=0):
n = {
'subject': 'eye_process.should_start.%s' % eye_id,
'eye_id': eye_id,
'delay': delay
}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id}
ipc_pub.notify(n)
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
ipc_pub.notify(n)
def handle_notifications(n):
subject = n['subject']
if subject == 'set_detection_mapping_mode':
if n['mode'] == '2d':
if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name:
logger.warning(
"The gaze mapper is not supported in 2d mode. Please recalibrate."
)
g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
g_pool.detection_mapping_mode = n['mode']
elif subject == 'start_plugin':
g_pool.plugins.add(plugin_by_name[n['name']],
args=n.get('args', {}))
elif subject == 'eye_process.started':
n = {
'subject': 'set_detection_mapping_mode',
'mode': g_pool.detection_mapping_mode
}
ipc_pub.notify(n)
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': g_pool.app,
'doc': world.__doc__
})
for p in g_pool.plugins:
if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify:
ipc_pub.notify({
'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__
})
#window and gl setup
glfw.glfwInit()
width, height = session_settings.get('window_size', cap.frame_size)
main_window = glfw.glfwCreateWindow(width, height, "World")
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-350, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('audio_mode', audio, selection=audio.audio_modes))
general_settings.append(
ui.Selector('detection_mapping_mode',
g_pool,
label='detection & mapping mode',
setter=set_detection_mapping_mode,
selection=['2d', '3d']))
general_settings.append(
ui.Switch('eye0_process',
label='Detect eye 0',
setter=lambda alive: start_stop_eye(0, alive),
getter=lambda: eyes_are_alive[0].value))
general_settings.append(
ui.Switch('eye1_process',
label='Detect eye 1',
setter=lambda alive: start_stop_eye(1, alive),
getter=lambda: eyes_are_alive[1].value))
selector_label = "Select to load"
labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins]
user_launchable_plugins.insert(0, selector_label)
labels.insert(0, selector_label)
general_settings.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=labels,
setter=open_plugin,
getter=lambda: selector_label))
general_settings.append(
ui.Info_Text('Capture Version: %s' % g_pool.version))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture.init_gui()
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(g_pool.quickbar)
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(
0,
ui.Selector(
'active_calibration_plugin',
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection=calibration_plugins,
labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
setter=open_plugin,
label='Method'))
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(
0,
ui.Selector('capture_manager',
setter=open_plugin,
getter=lambda: g_pool.capture_manager.__class__,
selection=manager_classes,
labels=[b.gui_name for b in manager_classes],
label='Manager'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
# refresh speed settings
glfw.glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil0_graph = graph.Bar_Graph(max_val=1.0)
pupil0_graph.pos = (260, 130)
pupil0_graph.update_rate = 5
pupil0_graph.label = "id0 conf: %0.2f"
pupil1_graph = graph.Bar_Graph(max_val=1.0)
pupil1_graph.pos = (380, 130)
pupil1_graph.update_rate = 5
pupil1_graph.label = "id1 conf: %0.2f"
pupil_graphs = pupil0_graph, pupil1_graph
if session_settings.get('eye1_process_alive', False):
launch_eye_process(1, delay=0.6)
if session_settings.get('eye0_process_alive', True):
launch_eye_process(0, delay=0.3)
ipc_pub.notify({'subject': 'world_process.started'})
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
g_pool.capture.on_notify(n)
for p in g_pool.plugins:
p.on_notify(n)
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except StreamError as e:
prev_settings = g_pool.capture.settings
g_pool.capture.deinit_gui()
g_pool.capture.cleanup()
g_pool.capture = None
prev_settings[
'info_text'] = "'%s' disconnected." % prev_settings['name']
g_pool.capture = Fake_Source(g_pool, **prev_settings)
g_pool.capture.init_gui()
ipc_pub.notify({'subject': 'recording.should_stop'})
logger.error("Error getting frame. Falling back to Fake source.")
logger.debug("Caught error: %s" % e)
sleep(.2)
continue
except EndofVideoFileError:
logger.warning("Video file is done. Rewinding")
g_pool.capture.seek_to_frame(0)
continue
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
recent_pupil_data = []
recent_gaze_data = []
while pupil_sub.new_data:
t, p = pupil_sub.recv()
pupil_graphs[p['id']].add(p['confidence'])
recent_pupil_data.append(p)
new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p)
for g in new_gaze_data:
gaze_pub.send('gaze', g)
recent_gaze_data += new_gaze_data
events['pupil_positions'] = recent_pupil_data
events['gaze_positions'] = recent_gaze_data
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
#send new events to ipc:
del events['pupil_positions'] #already on the wire
del events['gaze_positions'] #send earlier in this loop
del events['dt'] #no need to send this
for topic, data in events.iteritems():
assert (isinstance(data, (list, tuple)))
for d in data:
ipc_pub.send(topic, d)
# render camera image
glfw.glfwMakeContextCurrent(main_window)
if is_window_visible(main_window):
g_pool.image_tex.update_from_frame(frame)
glFlush()
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
if is_window_visible(main_window):
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil0_graph.draw()
pupil1_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture.deinit_gui()
g_pool.capture.cleanup()
#shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'world_process.stopped'})
#shut down launcher
n = {'subject': 'launcher_process.should_stop'}
ipc_pub.notify(n)
zmq_ctx.destroy()
def session(rec_dir):
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
user_launchable_plugins = [
Video_Export_Launcher, Raw_Data_Exporter, Vis_Circle, Vis_Cross,
Vis_Polyline, Vis_Light_Points, Vis_Fixation, Scan_Path,
Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
Vis_Watermark, Manual_Gaze_Correction, Show_Calibration,
Offline_Surface_Tracker, Batch_Exporter, Eye_Video_Overlay,
Annotation_Player, Log_History
] #,Marker_Auto_Trim_Marks
user_launchable_plugins += import_runtime_plugins(
os.path.join(user_dir, 'plugins'))
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
index_by_name = dict(zip(name_by_index, range(len(name_by_index))))
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'" % new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'%s' is not a valid pupil recording" %
new_rec_dir)
tick = delta_t()
def get_dt():
return next(tick)
update_recording_to_recent(rec_dir)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if f[-3:] in ('mp4', 'mkv', 'avi')
][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
rec_version = read_rec_version(meta_info)
app_version = get_version(version_file)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: %s' % app_version)
logger.info('System Info: %s' % get_system_info())
timestamps = np.load(timestamps_path)
# Initialize capture
cap = File_Capture(video_path, timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version",
VersionFormat('0.0')) < get_version(version_file):
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', (0, 0))
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
] #populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
logger.warning('play%s' % new_state)
if cap.get_frame_index() > cap.get_frame_count() - 1:
cap.seek_to_frame(1) #avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = slice(g_pool.trim_marks.in_mark,
g_pool.trim_marks.out_mark)
export_dir = os.path.join(
g_pool.rec_dir, 'exports',
'%s-%s' % (export_range.start, export_range.stop))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
logger.warning(
"Previous export for range [%s-%s] already exsits - overwriting."
% (export_range.start, export_range.stop))
else:
logger.info('Created export dir at "%s"' % export_dir)
notification = {
'subject': 'should_export',
'range': export_range,
'export_dir': export_dir
}
g_pool.notifications.append(notification)
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Growing_Menu("Settings",
pos=(-350, 20),
size=(300, 400))
g_pool.main_menu.append(
ui.Button("Close Pupil Player",
lambda: glfwSetWindowShouldClose(main_window, True)))
g_pool.main_menu.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=0.75,
max=2.5,
label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
rec_version))
selector_label = "Select to load"
labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins]
user_launchable_plugins.insert(0, selector_label)
labels.insert(0, selector_label)
g_pool.main_menu.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=labels,
setter=open_plugin,
getter=lambda: selector_label))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=unichr(0xf04b).encode('utf-8'),
setter=toggle_play,
hotkey=GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=unichr(0xf04e).encode('utf-8'),
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=unichr(0xf04a).encode('utf-8'),
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=unichr(0xf063).encode('utf-8'),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([
g_pool.play_button, g_pool.forward_button, g_pool.backward_button,
g_pool.export_button
])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Scan_Path', {}),
('Vis_Polyline', {}), ('Vis_Circle', {}),
('Video_Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
#trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_timestamp() - .03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(
g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(
g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
