def player(rec_dir, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, app_version):
# general imports
from time import sleep
import logging
from glob import glob
from time import time, strftime, localtime
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
# imports
from file_methods import Persistent_Dict, load_object, next_export_sub_dir
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture, RGBA
import gl_utils
# capture
from video_capture import init_playback_source, EndofVideoError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
from csv_utils import write_key_value_file
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_control import Seek_Control
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
# from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from batch_exporter import Batch_Exporter, Batch_Export
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
from system_graphs import System_Graphs
from system_timelines import System_Timelines
from blink_detection import Offline_Blink_Detection
from audio_playback import Audio_Playback
from imotions_exporter import iMotions_Exporter
assert VersionFormat(pyglui_version) >= VersionFormat('1.20'), 'pyglui out of date, please upgrade to newest version'
runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
system_plugins = [Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export, System_Timelines, Audio_Playback]
user_plugins = [Vis_Circle,
Vis_Fixation,
Vis_Polyline,
Vis_Light_Points,
Vis_Cross,
Vis_Watermark,
Vis_Eye_Video_Overlay,
Vis_Scan_Path,
Offline_Fixation_Detector,
Offline_Blink_Detection,
Batch_Exporter,
Video_Export_Launcher,
Offline_Surface_Tracker,
Raw_Data_Exporter,
Annotation_Player,
Log_History,
Pupil_From_Recording,
Offline_Pupil_Detection,
Gaze_From_Recording,
iMotions_Exporter,
Offline_Calibration
] + runtime_plugins
plugins = system_plugins + user_plugins
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal hdpi_factor
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
g_pool.camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
icon_bar_width = 50
window_size = None
hdpi_factor = 1.0
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg', '.fake')
video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in valid_ext][0]
init_playback_source(g_pool, timing='external', source_path=video_path)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version", '0.0')) != app_version:
logger.info("Session setting are a different version of this app. I will not use those.")
session_settings.clear()
width, height = g_pool.capture.frame_size
width += icon_bar_width
width, height = session_settings.get('window_size', (width, height))
window_pos = session_settings.get('window_position', window_position_default)
window_name = "Pupil Player: {} - {}".format(meta_info["Recording Name"],
os.path.split(rec_dir)[-1])
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, window_name, None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = (g_pool.camera_render_size[0] + int(icon_bar_width*g_pool.gui_user_scale*hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1])
logger.warning(icon_bar_width*g_pool.gui_user_scale*hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.min_calibration_confidence = session_settings.get('min_calibration_confidence', 0.8)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
left_idx = g_pool.seek_control.trim_left
right_idx = g_pool.seek_control.trim_right
export_range = left_idx, right_idx + 1 # exclusive range.stop
export_dir = os.path.join(g_pool.rec_dir, 'exports')
export_dir = next_export_sub_dir(export_dir)
os.makedirs(export_dir)
logger.info('Created export dir at "{}"'.format(export_dir))
export_info = {'Player Software Version': str(g_pool.version),
'Data Format Version': meta_info['Data Format Version'],
'Export Date': strftime("%d.%m.%Y", localtime()),
'Export Time': strftime("%H:%M:%S", localtime()),
'Frame Index Range:': g_pool.seek_control.get_frame_index_trim_range_string(),
'Relative Time Range': g_pool.seek_control.get_rel_time_trim_range_string(),
'Absolute Time Range': g_pool.seek_control.get_abs_time_trim_range_string()}
with open(os.path.join(export_dir, 'export_info.csv'), 'w') as csv:
write_key_value_file(csv, export_info)
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
ipc_pub.notify({'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2.})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left')
g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden')
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.),
size=(0., 0.), header_pos='headline')
g_pool.user_timelines.color = RGBA(a=0.)
g_pool.user_timelines.collapsed = True
# add container that constaints itself to the seekbar height
vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
vert_constr.append(g_pool.user_timelines)
g_pool.timelines.append(vert_constr)
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
general_settings = ui.Growing_Menu('General', header_pos='headline')
general_settings.append(ui.Button('Reset window size', set_window_size))
general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2]+list(np.arange(1.5, 5.1, .5)), label='Interface Size'))
general_settings.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
general_settings.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
general_settings.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
general_settings.append(ui.Info_Text('High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence.'))
general_settings.append(ui.Slider('min_data_confidence', g_pool,
setter=set_data_confidence,
step=.05, min=0.0, max=1.0,
label='Minimum data confidence'))
general_settings.append(ui.Button('Restart with default settings', reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons')
icon.tooltip = 'General Settings'
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100))
g_pool.export_button = ui.Thumb('export',
label=chr(0xe2c5),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='pupil_icons')
g_pool.quickbar.extend([g_pool.export_button])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [('Plugin_Manager', {}),
('Seek_Control', {}),
('Log_Display', {}),
('Raw_Data_Exporter', {}),
('Vis_Polyline', {}),
('Vis_Circle', {}),
('System_Graphs', {}),
('System_Timelines', {}),
('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}),
('Gaze_From_Recording', {}),
('Audio_Playback', {})]
g_pool.plugins = Plugin_List(g_pool, session_settings.get('loaded_plugins', default_plugins))
# Manually add g_pool.capture to the plugin list
g_pool.plugins._plugins.append(g_pool.capture)
g_pool.plugins._plugins.sort(key=lambda p: p.order)
g_pool.capture.init_ui()
general_settings.insert(-1, ui.Text_Input('rel_time_trim_section',
getter=g_pool.seek_control.get_rel_time_trim_range_string,
setter=g_pool.seek_control.set_rel_time_trim_range_string,
label='Relative time range to export'))
general_settings.insert(-1, ui.Text_Input('frame_idx_trim_section',
getter=g_pool.seek_control.get_frame_index_trim_range_string,
setter=g_pool.seek_control.set_frame_index_trim_range_string,
label='Frame index range to export'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
toggle_general_settings(True)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(
g_pool.plugin_by_name[n['name']], args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
events = {}
# report time between now and the last loop interation
events['dt'] = get_dt()
# pupil and gaze positions are added by their respective producer plugins
events['pupil_positions'] = []
events['gaze_positions'] = []
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ''
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode())
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in user_input.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in user_input.chars:
for p in g_pool.plugins:
p.on_char(char_)
# present frames at appropriate speed
g_pool.seek_control.wait(events['frame'].timestamp)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['min_calibration_confidence'] = g_pool.min_calibration_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings['window_size'] = session_window_size
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.capture.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except:
import traceback
trace = traceback.format_exc()
logger.error('Process Player crashed with trace:\n{}'.format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
sleep(1.0)
def eye(
timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None,
):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
if is_alive_flag.value:
# indicates eye process that this is a duplicated startup
logger.warning("Aborting redundant eye process startup")
return
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
# general imports
import traceback
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible, glViewport
from ui_roi import UIRoi
# monitoring
import psutil
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, AV_Writer
from ndsi import H264Writer
from video_capture import source_classes
from video_capture import manager_classes
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy
pupil_detectors = {
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D,
Detector_Dummy.__name__: Detector_Dummy,
}
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id + 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (600, 90 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
icon_bar_width = 50
window_size = None
camera_render_size = None
hdpi_factor = 1.0
# g_pool holds variables for this process
g_pool = SimpleNamespace()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = "capture"
g_pool.process = "eye{}".format(eye_id)
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x *= hdpi_factor
y *= hdpi_factor
g_pool.gui.update_mouse(x, y)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, g_pool.capture.frame_size)
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
plugins = (g_pool.capture_manager, g_pool.capture)
# call `on_drop` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(p.on_drop(paths) for p in plugins)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir,
"user_settings_eye{}".format(eye_id)))
if VersionFormat(session_settings.get("version",
"0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.capture_manager = None
g_pool.flip = session_settings.get("flip", False)
g_pool.display_mode = session_settings.get("display_mode",
"camera_image")
g_pool.display_mode_info_text = {
"camera_image":
"Raw eye camera image. This uses the least amount of CPU power",
"roi":
"Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
"algorithm":
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
}
capture_manager_settings = session_settings.get(
"capture_manager_settings", ("UVC_Manager", {}))
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__: c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](
g_pool, **manager_settings)
if eye_id == 0:
cap_src = [
"Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"
]
else:
cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"]
# Initialize capture
default_settings = (
"UVC_Source",
{
"preferred_names": cap_src,
"frame_size": (320, 240),
"frame_rate": 120
},
)
capture_source_settings = overwrite_cap_settings or session_settings.get(
"capture_settings", default_settings)
source_class_name, source_settings = capture_source_settings
source_class_by_name = {c.__name__: c for c in source_classes}
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings)
assert g_pool.capture
g_pool.u_r = UIRoi(
(g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get("roi")
if roi_user_settings and tuple(
roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
pupil_detector_settings = session_settings.get(
"pupil_detector_settings", None)
last_pupil_detector = pupil_detectors[session_settings.get(
"last_pupil_detector", Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,
pupil_detector_settings)
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_ui()
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# Initialize glfw
glfw.glfwInit()
title = "Pupil Capture - eye {}".format(eye_id)
width, height = g_pool.capture.frame_size
width *= 2
height *= 2
width += icon_bar_width
width, height = session_settings.get("window_size", (width, height))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get("window_position",
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(
np.ones((1, 1), dtype=np.uint8) + 125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos="left")
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos="hidden")
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2],
label="Interface Size",
))
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width *= 2
f_height *= 2
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
def uroi_on_mouse_button(button, action, mods):
if g_pool.display_mode == "roi":
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
x, y = glfw.glfwGetCursorPos(main_window)
# pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
x *= hdpi_factor
y *= hdpi_factor
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(
pos,
g_pool.capture.frame_size) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,
g_pool.u_r.handle_size,
g_pool.u_r.handle_size):
# if the roi interacts we dont want
# the gui to interact as well
return
general_settings.append(ui.Button("Reset window size",
set_window_size))
general_settings.append(
ui.Switch("flip", g_pool, label="Flip image display"))
general_settings.append(
ui.Selector(
"display_mode",
g_pool,
setter=set_display_mode_info,
selection=["camera_image", "roi", "algorithm"],
labels=["Camera Image", "ROI", "Algorithm"],
label="Mode",
))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
detector_selector = ui.Selector(
"pupil_detector",
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Detector_Dummy, Detector_2D, Detector_3D],
labels=["disabled", "C++ 2d detector", "C++ 3d detector"],
label="Detection method",
)
general_settings.append(detector_selector)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
toggle_general_settings(False)
g_pool.pupil_detector.init_ui()
g_pool.capture.init_ui()
g_pool.capture_manager.init_ui()
g_pool.writer = None
def replace_source(source_class_name, source_settings):
g_pool.capture.deinit_ui()
g_pool.capture.cleanup()
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings)
g_pool.capture.init_ui()
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
g_pool.replace_source = replace_source # for ndsi capture
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 50)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 50)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = "jpeg"
frame_publish_format_recent_warning = False
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning("Process started.")
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification["subject"]
if subject.startswith("eye_process.should_stop"):
if notification["eye_id"] == eye_id:
break
elif subject == "set_detection_mapping_mode":
if notification["mode"] == "3d":
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
elif notification["mode"] == "2d":
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
else:
if not isinstance(g_pool.pupil_detector,
Detector_Dummy):
set_detector(Detector_Dummy)
detector_selector.read_only = True
elif subject == "recording.started":
if notification["record_eye"] and g_pool.capture.online:
record_path = notification["rec_path"]
raw_mode = notification["compression"]
logger.info(
"Will save eye video to: {}".format(record_path))
video_path = os.path.join(record_path,
"eye{}.mp4".format(eye_id))
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(
video_path, g_pool.capture.frame_rate)
elif hasattr(g_pool.capture._recent_frame,
"h264_buffer"):
g_pool.writer = H264Writer(
video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate,
)
else:
g_pool.writer = AV_Writer(
video_path, g_pool.capture.frame_rate)
elif subject == "recording.stopped":
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
elif subject.startswith("meta.should_doc"):
ipc_socket.notify({
"subject": "meta.doc",
"actor": "eye{}".format(eye_id),
"doc": eye.__doc__,
})
elif subject.startswith("frame_publishing.started"):
should_publish_frames = True
frame_publish_format = notification.get("format", "jpeg")
elif subject.startswith("frame_publishing.stopped"):
should_publish_frames = False
frame_publish_format = "jpeg"
elif (subject.startswith("start_eye_capture")
and notification["target"] == g_pool.process):
replace_source(notification["name"], notification["args"])
elif notification["subject"].startswith(
"pupil_detector.set_property"):
target_process = notification.get("target", g_pool.process)
should_apply = target_process == g_pool.process
if should_apply:
try:
property_name = notification["name"]
property_value = notification["value"]
if "2d" in notification["subject"]:
g_pool.pupil_detector.set_2d_detector_property(
property_name, property_value)
elif "3d" in notification["subject"]:
if not isinstance(g_pool.pupil_detector,
Detector_3D):
raise ValueError(
"3d properties are only available"
" if 3d detector is active")
g_pool.pupil_detector.set_3d_detector_property(
property_name, property_value)
else:
raise KeyError("Notification subject does not "
"specifiy detector type.")
logger.debug("`{}` property set to {}".format(
property_name, property_value))
except KeyError:
logger.error("Malformed notification received")
logger.debug(traceback.format_exc())
except (ValueError, TypeError):
logger.error("Invalid property or value")
logger.debug(traceback.format_exc())
elif notification["subject"].startswith(
"pupil_detector.broadcast_properties"):
target_process = notification.get("target", g_pool.process)
should_respond = target_process == g_pool.process
if should_respond:
props = g_pool.pupil_detector.get_detector_properties()
properties_broadcast = {
"subject":
"pupil_detector.properties.{}".format(eye_id),
**props, # add properties to broadcast
}
ipc_socket.notify(properties_broadcast)
g_pool.capture.on_notify(notification)
g_pool.capture_manager.on_notify(notification)
# Get an image from the grabber
event = {}
g_pool.capture.recent_events(event)
frame = event.get("frame")
g_pool.capture_manager.recent_events(event)
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
f_height,
f_width,
):
g_pool.pupil_detector.on_resolution_change(
(g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
g_pool.capture.frame_size,
)
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
assert data is not None
except (AttributeError, AssertionError, NameError):
if not frame_publish_format_recent_warning:
frame_publish_format_recent_warning = True
logger.warning(
'{}s are not compatible with format "{}"'.
format(type(frame), frame_publish_format))
else:
frame_publish_format_recent_warning = False
pupil_socket.send({
"topic":
"frame.eye.{}".format(eye_id),
"width":
frame.width,
"height":
frame.height,
"index":
frame.index,
"timestamp":
frame.timestamp,
"format":
frame_publish_format,
"__raw_data__": [data],
})
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1.0 / dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == "algorithm")
if result is not None:
result["id"] = eye_id
result["topic"] = "pupil.{}".format(eye_id)
pupil_socket.send(result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
if frame:
# switch to work in normalized coordinate space
if g_pool.display_mode == "algorithm":
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ("camera_image", "roi"):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
glViewport(0, 0, *camera_render_size)
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
f_width, f_height = g_pool.capture.frame_size
make_coord_system_pixel_based((f_height, f_width, 3),
g_pool.flip)
if frame and result:
if result["method"] == "3d c++":
eye_ball = result["projected_sphere"]
try:
pts = cv2.ellipse2Poly(
(
int(eye_ball["center"][0]),
int(eye_ball["center"][1]),
),
(
int(eye_ball["axes"][0] / 2),
int(eye_ball["axes"][1] / 2),
),
int(eye_ball["angle"]),
0,
360,
8,
)
except ValueError as e:
pass
else:
draw_polyline(
pts,
2,
RGBA(0.0, 0.9, 0.1,
result["model_confidence"]),
)
if result["confidence"] > 0:
if "ellipse" in result:
pts = cv2.ellipse2Poly(
(
int(result["ellipse"]["center"][0]),
int(result["ellipse"]["center"][1]),
),
(
int(result["ellipse"]["axes"][0] / 2),
int(result["ellipse"]["axes"][1] / 2),
),
int(result["ellipse"]["angle"]),
0,
360,
15,
)
confidence = result["confidence"] * 0.7
draw_polyline(pts, 1,
RGBA(1.0, 0, 0, confidence))
draw_points(
[result["ellipse"]["center"]],
size=20,
color=RGBA(1.0, 0.0, 0.0, confidence),
sharpness=1.0,
)
glViewport(0, 0, *camera_render_size)
glfw.glfwPollEvents()
make_coord_system_pixel_based((f_height, f_width, 3),
g_pool.flip)
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == "roi":
g_pool.u_r.draw_points(g_pool.gui.scale)
glViewport(0, 0, *window_size)
make_coord_system_pixel_based((*window_size[::-1], 3),
g_pool.flip)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
unused_elements = g_pool.gui.update()
for butt in unused_elements.buttons:
uroi_on_mouse_button(*butt)
make_coord_system_pixel_based((*window_size[::-1], 3),
g_pool.flip)
g_pool.pupil_detector.visualize(
) # detector decides if we visualize or not
# update screen
glfw.glfwSwapBuffers(main_window)
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer = None
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
# save session persistent settings
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["roi"] = g_pool.u_r.get()
session_settings["flip"] = g_pool.flip
session_settings["display_mode"] = g_pool.display_mode
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["capture_settings"] = (
g_pool.capture.class_name,
g_pool.capture.get_init_dict(),
)
session_settings["capture_manager_settings"] = (
g_pool.capture_manager.class_name,
g_pool.capture_manager.get_init_dict(),
)
session_settings["window_position"] = glfw.glfwGetWindowPos(
main_window)
session_settings["version"] = str(g_pool.version)
session_settings[
"last_pupil_detector"] = g_pool.pupil_detector.__class__.__name__
session_settings[
"pupil_detector_settings"] = g_pool.pupil_detector.get_settings()
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
g_pool.capture.deinit_ui()
g_pool.capture_manager.deinit_ui()
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
glfw.glfwDestroyWindow(main_window)
g_pool.gui.terminate()
glfw.glfwTerminate()
logger.info("Process shutting down.")
import zmq_tools
import zmq
ctx = zmq.Context()
requester = ctx.socket(zmq.REQ)
requester.connect('tcp://127.0.1.1:50020') #change ip if using remote machine
requester.send_string('SUB_PORT')
ipc_sub_port = requester.recv()
topics_list = ('notify.eye_process.', 'notify.player_process.',
'notify.world_process.')
print('tcp://127.0.1.1:' + ipc_sub_port.decode())
monitor = zmq_tools.Msg_Receiver(ctx,
'tcp://127.0.1.1:%s' % ipc_sub_port.decode(),
topics=topics_list)
while True:
print(monitor.recv())
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, version):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``
``eye_process.started``
``start_plugin``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events
that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
# general imports
import logging
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# display
import glfw
from pyglui import ui, graph, cygl, __version__ as pyglui_version
assert pyglui_version >= '1.7'
from pyglui.cygl.utils import Named_Texture
import gl_utils
# monitoring
import psutil
# helpers/utils
from version_utils import VersionFormat
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info, timer
from uvc import get_time_monotonic
logger.info('Application Version: {}'.format(version))
logger.info('System Info: {}'.format(get_system_info()))
import audio
# trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin
from fixation_detector import Fixation_Detector
from recorder import Recorder
from display_recent_gaze import Display_Recent_Gaze
from time_sync import Time_Sync
from pupil_remote import Pupil_Remote
from pupil_groups import Pupil_Groups
from surface_tracker import Surface_Tracker
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
from frame_publisher import Frame_Publisher
from blink_detection import Blink_Detection
from video_capture import source_classes, manager_classes, Base_Manager
from pupil_data_relay import Pupil_Data_Relay
from remote_recorder import Remote_Recorder
from audio_capture import Audio_Capture
from accuracy_visualizer import Accuracy_Visualizer
# UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (0, 0)
elif platform.system() == 'Windows':
scroll_factor = 10.0
window_position_default = (8, 31)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
# g_pool holds variables for this process they are accesible to all plugins
g_pool = Global_Container()
g_pool.app = 'capture'
g_pool.process = 'world'
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eyes_are_alive = eyes_are_alive
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins'))
calibration_plugins += [p for p in runtime_plugins if issubclass(p, Calibration_Plugin)]
runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Calibration_Plugin)]
manager_classes += [p for p in runtime_plugins if issubclass(p, Base_Manager)]
runtime_plugins = [p for p in runtime_plugins if not issubclass(p, Base_Manager)]
user_launchable_plugins = [Audio_Capture, Pupil_Groups, Frame_Publisher, Pupil_Remote, Time_Sync, Surface_Tracker,
Annotation_Capture, Log_History, Fixation_Detector,
Blink_Detection, Remote_Recorder, Accuracy_Visualizer] + runtime_plugins
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay]
plugin_by_index = (system_plugins + user_launchable_plugins + calibration_plugins
+ gaze_mapping_plugins + manager_classes + source_classes)
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_capture_settings = {
'preferred_names': ["Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)",
"C510", "B525", "C525", "C615", "C920", "C930e"],
'frame_size': (1280, 720),
'frame_rate': 30
}
default_plugins = [("UVC_Source", default_capture_settings),
('Pupil_Data_Relay', {}),
('UVC_Manager', {}),
('Log_Display', {}),
('Dummy_Gaze_Mapper', {}),
('Display_Recent_Gaze', {}),
('Screen_Marker_Calibration', {}),
('Recorder', {}),
('Pupil_Remote', {})]
# Callback functions
def on_resize(window, w, h):
if gl_utils.is_window_visible(window):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(
window)[0] / glfw.glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode('utf-8') for x in range(count)]
for p in g_pool.plugins:
p.on_drop(paths)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_world'))
if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
logger.info("Session setting are from a different version of this app. I will not use those.")
session_settings.clear()
g_pool.iconified = False
g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '3d')
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
g_pool.capture_manager = None
audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def launch_eye_process(eye_id, delay=0):
n = {'subject': 'eye_process.should_start.{}'.format(eye_id),
'eye_id': eye_id, 'delay': delay}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id,'delay':0.2}
ipc_pub.notify(n)
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
ipc_pub.notify(n)
def handle_notifications(n):
subject = n['subject']
if subject == 'set_detection_mapping_mode':
if n['mode'] == '2d':
if ("Vector_Gaze_Mapper" in
g_pool.active_gaze_mapping_plugin.class_name):
logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
g_pool.detection_mapping_mode = n['mode']
elif subject == 'start_plugin':
g_pool.plugins.add(plugin_by_name[n['name']], args=n.get('args', {}))
elif subject == 'stop_plugin':
for p in g_pool.plugins:
if p.class_name == n['name']:
p.alive = False
g_pool.plugins.clean()
elif subject == 'eye_process.started':
n = {'subject': 'set_detection_mapping_mode',
'mode': g_pool.detection_mapping_mode}
ipc_pub.notify(n)
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({'subject': 'meta.doc',
'actor': g_pool.app,
'doc': world.__doc__})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__})
# window and gl setup
glfw.glfwInit()
width, height = session_settings.get('window_size', (1280, 720))
main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World")
window_pos = session_settings.get('window_position', window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def reset_restart():
logger.warning("Resetting all settings and restarting Capture.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({'subject': 'reset_restart_process.should_start'})
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.sidebar = ui.Scrolling_Menu("Settings", pos=(-350, 0), size=(0, 0), header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Button('Reset to default settings',reset_restart))
general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface size'))
general_settings.append(ui.Button('Reset window size', lambda: glfw.glfwSetWindowSize(main_window,g_pool.capture.frame_size[0],g_pool.capture.frame_size[1])) )
general_settings.append(ui.Selector('audio_mode', audio, selection=audio.audio_modes))
general_settings.append(ui.Selector('detection_mapping_mode',
g_pool,
label='detection & mapping mode',
setter=set_detection_mapping_mode,
selection=['2d','3d']
))
general_settings.append(ui.Switch('eye0_process',
label='Detect eye 0',
setter=lambda alive: start_stop_eye(0,alive),
getter=lambda: eyes_are_alive[0].value
))
general_settings.append(ui.Switch('eye1_process',
label='Detect eye 1',
setter=lambda alive: start_stop_eye(1,alive),
getter=lambda: eyes_are_alive[1].value
))
selector_label = "Select to load"
labels = [p.__name__.replace('_', ' ') for p in user_launchable_plugins]
user_launchable_plugins.insert(0, selector_label)
labels.insert(0, selector_label)
general_settings.append(ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=labels,
setter=open_plugin,
getter=lambda: selector_label))
general_settings.append(ui.Info_Text('Capture Version: {}'.format(g_pool.version)))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture_source_menu.collapsed = True
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.collapsed = True
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(g_pool.quickbar)
# plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, session_settings.get('loaded_plugins', default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(0,ui.Selector(
'active_calibration_plugin',
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= open_plugin,label='Method'
))
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(0,ui.Selector(
'capture_manager',
setter = open_plugin,
getter = lambda: g_pool.capture_manager.__class__,
selection = manager_classes,
labels = [b.gui_name for b in manager_classes],
label = 'Manager'
))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil0_graph = graph.Bar_Graph(max_val=1.0)
pupil0_graph.pos = (260, 130)
pupil0_graph.update_rate = 5
pupil0_graph.label = "id0 conf: %0.2f"
pupil1_graph = graph.Bar_Graph(max_val=1.0)
pupil1_graph.pos = (380, 130)
pupil1_graph.update_rate = 5
pupil1_graph.label = "id1 conf: %0.2f"
pupil_graphs = pupil0_graph, pupil1_graph
g_pool.graphs = [cpu_graph, fps_graph, pupil0_graph, pupil1_graph]
# trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
if session_settings.get('eye1_process_alive', False):
launch_eye_process(1, delay=0.6)
if session_settings.get('eye0_process_alive', True):
launch_eye_process(0, delay=0.3)
ipc_pub.notify({'subject': 'world_process.started'})
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
#a dictionary that allows plugins to post and read events
events = {}
# report time between now and the last loop interation
events['dt'] = get_dt()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# update performace graphs
if 'frame' in events:
t = events["frame"].timestamp
dt, ts = t-ts, t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
for p in events["pupil_positions"]:
pupil_graphs[p['id']].add(p['confidence'])
cpu_graph.update()
# send new events to ipc:
del events['pupil_positions'] # already on the wire
del events['gaze_positions'] # sent earlier
if 'frame' in events:
del events['frame'] # send explicity with frame publisher
if 'depth_frame' in events:
del events['depth_frame']
if 'audio_packets' in events:
del events['audio_packets']
del events['dt'] # no need to send this
for topic, data in events.items():
assert(isinstance(data, (list, tuple)))
for d in data:
ipc_pub.send(topic, d)
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
if window_should_update() and gl_utils.is_window_visible(main_window):
for p in g_pool.plugins:
p.gl_display()
for g in g_pool.graphs:
g.draw()
unused_elements = g_pool.gui.update()
for button, action, mods in unused_elements.buttons:
pos = glfw.glfwGetCursorPos(main_window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in unused_elements.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in unused_elements.chars:
for p in g_pool.plugins:
p.on_char(char_)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
# shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'world_process.stopped'})
def launcher():
"""Starts eye processes. Hosts the IPC Backbone and Logging functions.
Reacts to notifications:
``launcher_process.should_stop``: Stops the launcher process
``eye_process.should_start``: Starts the eye process
"""
## IPC
#shared values
timebase = Value(c_double, 0)
eyes_are_alive = Value(c_bool, 0), Value(c_bool, 0)
#network backbone setup
zmq_ctx = zmq.Context()
# lets get some open ports:
test_socket = zmq_ctx.socket(zmq.SUB)
ipc_sub_port = test_socket.bind_to_random_port('tcp://*',
min_port=5001,
max_port=6000,
max_tries=100)
ipc_pub_port = test_socket.bind_to_random_port('tcp://*',
min_port=6001,
max_port=7000,
max_tries=100)
ipc_push_port = test_socket.bind_to_random_port('tcp://*',
min_port=6001,
max_port=7000,
max_tries=100)
test_socket.close(linger=0)
ipc_pub_url = 'tcp://127.0.0.1:%s' % ipc_pub_port
ipc_sub_url = 'tcp://127.0.0.1:%s' % ipc_sub_port
ipc_push_url = 'tcp://127.0.0.1:%s' % ipc_push_port
#We use a zmq forwarder and the zmq PUBSUB pattern to do all our IPC.
def ipc_backbone(in_url, out_url):
ctx = zmq.Context.instance()
xsub = ctx.socket(zmq.XSUB)
xsub.bind(in_url)
xpub = ctx.socket(zmq.XPUB)
xpub.bind(out_url)
try:
zmq.proxy(xsub, xpub)
except zmq.ContextTerminated:
xsub.close()
xpub.close()
#reliable msg dispatch to the IPC via push bridge
def pull_pub(in_url, out_url):
ctx = zmq.Context.instance()
pull = ctx.socket(zmq.PULL)
pull.bind(in_url)
pub = ctx.socket(zmq.PUB)
pub.connect(out_url)
while True:
m = pull.recv_multipart()
pub.send_multipart(m)
#The delay proxy handles delayed notififications.
def delay_proxy(in_url, out_url):
ctx = zmq.Context.instance()
sub = zmq_tools.Msg_Receiver(ctx, in_url, ('delayed_notify', ))
pub = zmq_tools.Msg_Dispatcher(ctx, out_url)
poller = zmq.Poller()
poller.register(sub.socket, zmq.POLLIN)
waiting_notifications = {}
try:
while True:
if poller.poll(timeout=250):
#Recv new delayed notification and store it.
topic, n = sub.recv()
n['_notify_time_'] = time() + n['delay']
waiting_notifications[n['subject']] = n
#When a notifications time has come, pop from dict and send it as notification
for n in waiting_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del n['delay']
del waiting_notifications[n['subject']]
pub.notify(n)
except zmq.ContextTerminated:
sub.close()
pub.close()
#recv log records from other processes.
def log_loop(ipc_sub_url, log_level_debug):
import logging
#Get the root logger
logger = logging.getLogger()
#set log level
if log_level_debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
#Stream to file
fh = logging.FileHandler(os.path.join(user_dir, 'capture.log'),
mode='w')
fh.setFormatter(
logging.Formatter(
'%(asctime)s - %(processName)s - [%(levelname)s] %(name)s: %(message)s'
))
logger.addHandler(fh)
#Stream to console.
ch = logging.StreamHandler()
ch.setFormatter(
logging.Formatter(
'%(processName)s - [%(levelname)s] %(name)s: %(message)s'))
logger.addHandler(ch)
# IPC setup to receive log messages. Use zmq_tools.ZMQ_handler to send messages to here.
sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("logging", ))
while True:
topic, msg = sub.recv()
record = logging.makeLogRecord(msg)
logger.handle(record)
ipc_backbone_thread = Thread(target=ipc_backbone,
args=('tcp://*:%s' % ipc_pub_port,
'tcp://*:%s' % ipc_sub_port))
ipc_backbone_thread.setDaemon(True)
ipc_backbone_thread.start()
log_thread = Thread(target=log_loop,
args=(ipc_sub_url, 'debug' in sys.argv))
log_thread.setDaemon(True)
log_thread.start()
delay_thread = Thread(target=delay_proxy, args=(ipc_sub_url, ipc_push_url))
delay_thread.setDaemon(True)
delay_thread.start()
pull_pub = Thread(target=pull_pub,
args=('tcp://*:%s' % ipc_push_port, ipc_pub_url))
pull_pub.setDaemon(True)
pull_pub.start()
topics = ('notify.eye_process.', 'notify.launcher_process.',
'notify.meta.should_doc')
cmd_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=topics)
cmd_push = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
if app == 'service':
Process(target=service,
name='service',
args=(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, app_version, None)).start()
else:
Process(target=world,
name='world',
args=(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, app_version,
video_sources['world'])).start()
while True:
#block and listen for relevant messages.
topic, n = cmd_sub.recv()
if "notify.eye_process.should_start" in topic:
eye_id = n['eye_id']
if not eyes_are_alive[eye_id].value:
Process(target=eye,
name='eye%s' % eye_id,
args=(timebase, eyes_are_alive[eye_id], ipc_pub_url,
ipc_sub_url, ipc_push_url, user_dir, app_version,
eye_id,
video_sources['eye%s' % eye_id])).start()
elif "notify.launcher_process.should_stop" in topic:
break
elif "notify.meta.should_doc" in topic:
cmd_push.notify({
'subject': 'meta.doc',
'actor': 'launcher',
'doc': launcher.__doc__
})
for p in active_children():
p.join()
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, version, eye_id, overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
# general imports
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible, glViewport
from ui_roi import UIRoi
# monitoring
import psutil
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, AV_Writer
from ndsi import H264Writer
from video_capture import source_classes
from video_capture import manager_classes
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D}
# UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id + 30)
elif platform.system() == 'Windows':
scroll_factor = 10.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
icon_bar_width = 50
window_size = None
camera_render_size = None
# g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.process = 'eye{}'.format(eye_id)
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
# pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = normalize(pos, camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(pos,g_pool.capture.frame_size) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos, g_pool.u_r.handle_size + 40,g_pool.u_r.handle_size + 40):
# if the roi interacts we dont want
# the gui to interact as well
return
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = glfw.glfwGetFramebufferSize(
window)[0] / glfw.glfwGetWindowSize(window)[0]
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,g_pool.capture.frame_size )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode('utf-8') for x in range(count)]
g_pool.capture_manager.on_drop(paths)
g_pool.capture.on_drop(paths)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_eye{}'.format(eye_id)))
if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
logger.info("Session setting are from a different version of this app. I will not use those.")
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get(
'display_mode', 'camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager',{}))
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__:c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
if eye_id == 0:
cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
else:
cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]
# Initialize capture
default_settings = ('UVC_Source',{
'preferred_names' : cap_src,
'frame_size': (640,480),
'frame_rate': 90
})
capture_source_settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
source_class_name, source_settings = capture_source_settings
source_class_by_name = {c.__name__:c for c in source_classes}
g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
assert g_pool.capture
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(
g_pool, pupil_detector_settings)
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_ui()
def toggle_general_settings(collapsed):
#this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# Initialize glfw
glfw.glfwInit()
title = "Pupil Capture - eye {}".format(eye_id)
width, height = session_settings.get(
'window_size', g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get(
'window_position', window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(np.ones((1,1),dtype=np.uint8)+125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left')
g_pool.iconbar = ui.Scrolling_Menu("Icons",pos=(-icon_bar_width,0),size=(0,0),header_pos='hidden')
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
general_settings = ui.Growing_Menu('General',header_pos='headline')
general_settings.append(ui.Selector('gui_user_scale', g_pool,
setter=set_scale,
selection=[.8, .9, 1., 1.1, 1.2],
label='Interface Size'))
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width*g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
general_settings.append(ui.Button('Reset window size', set_window_size))
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image','roi','algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode")
)
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
detector_selector = ui.Selector('pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector, selection=[
Detector_2D, Detector_3D],
labels=['C++ 2d detector',
'C++ 3d detector'],
label="Detection method")
general_settings.append(detector_selector)
g_pool.menubar.append(general_settings)
icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons')
icon.tooltip = 'General Settings'
g_pool.iconbar.append(icon)
toggle_general_settings(False)
g_pool.pupil_detector.init_ui()
g_pool.capture.init_ui()
g_pool.capture_manager.init_ui()
g_pool.writer = None
def replace_source(source_class_name, source_settings):
g_pool.capture.deinit_ui()
g_pool.capture.cleanup()
g_pool.capture = source_class_by_name[source_class_name](g_pool,**source_settings)
g_pool.capture.init_ui()
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
g_pool.replace_source = replace_source # for ndsi capture
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 50)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 50)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = 'jpeg'
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification['subject']
if subject.startswith('eye_process.should_stop'):
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye'] and g_pool.capture.online:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: {}".format(record_path))
video_path = os.path.join(record_path, "eye{}.mp4".format(eye_id))
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(video_path, g_pool.capture.frame_rate)
elif hasattr(g_pool.capture._recent_frame, 'h264_buffer'):
g_pool.writer = H264Writer(video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate)
else:
g_pool.writer = AV_Writer(video_path, g_pool.capture.frame_rate)
elif subject == 'recording.stopped':
if g_pool.writer:
logger.info("Done recording.")
g_pool.writer.release()
g_pool.writer = None
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject': 'meta.doc',
'actor': 'eye{}'.format(eye_id),
'doc': eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format', 'jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
elif subject.startswith('start_eye_capture') and notification['target'] == g_pool.process:
replace_source(notification['name'],notification['args'])
g_pool.capture.on_notify(notification)
# Get an image from the grabber
event = {}
g_pool.capture.recent_events(event)
frame = event.get('frame')
g_pool.capture_manager.recent_events(event)
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (f_height, f_width):
g_pool.pupil_detector.on_resolution_change((g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]), g_pool.capture.frame_size)
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
else:
raise AttributeError()
except AttributeError:
pass
else:
pupil_socket.send('frame.eye.%s'%eye_id,{
'width': frame.width,
'height': frame.height,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s'%eye_id,result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
if frame:
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
glViewport(0, 0, *camera_render_size)
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
f_width, f_height = g_pool.capture.frame_size
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
if frame:
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly(
(int(eye_ball['center'][0]),
int(eye_ball['center'][1])),
(int(eye_ball['axes'][0] / 2),
int(eye_ball['axes'][1] / 2)),
int(eye_ball['angle']), 0, 360, 8)
except ValueError as e:
pass
else:
draw_polyline(pts, 2, RGBA(0., .9, .1, result['model_confidence']))
if result['confidence'] > 0:
if 'ellipse' in result:
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
confidence = result['confidence'] * 0.7
draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., confidence),
sharpness=1.)
glViewport(0, 0, *window_size)
make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
g_pool.gui.update()
glViewport(0, 0, *camera_render_size)
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
glViewport(0, 0, *window_size)
make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() # detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer = None
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.class_name, g_pool.capture.get_init_dict()
session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_ui()
g_pool.capture_manager.deinit_ui()
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
glfw.glfwDestroyWindow(main_window)
g_pool.gui.terminate()
glfw.glfwTerminate()
logger.info("Process shutting down.")
def world(
timebase,
eye_procs_alive,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
preferred_remote_port,
):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``
``eye_process.started``
``start_plugin``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events
that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
# general imports
from time import sleep
import logging
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
def launch_eye_process(eye_id, delay=0):
n = {
"subject": "eye_process.should_start.{}".format(eye_id),
"eye_id": eye_id,
"delay": delay,
}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {
"subject": "eye_process.should_stop.{}".format(eye_id),
"eye_id": eye_id,
"delay": 0.2,
}
ipc_pub.notify(n)
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
n = {"subject": "set_detection_mapping_mode", "mode": new_mode}
ipc_pub.notify(n)
try:
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
from tasklib.background.patches import IPCLoggingPatch
IPCLoggingPatch.ipc_push_url = ipc_push_url
# display
import glfw
from version_utils import VersionFormat
from pyglui import ui, cygl, __version__ as pyglui_version
assert VersionFormat(pyglui_version) >= VersionFormat(
"1.24"), "pyglui out of date, please upgrade to newest version"
from pyglui.cygl.utils import Named_Texture
import gl_utils
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info, timer
from uvc import get_time_monotonic
logger.info("Application Version: {}".format(version))
logger.info("System Info: {}".format(get_system_info()))
import audio
# trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import (
Plugin,
System_Plugin_Base,
Plugin_List,
import_runtime_plugins,
)
from plugin_manager import Plugin_Manager
from calibration_routines import (
calibration_plugins,
gaze_mapping_plugins,
Calibration_Plugin,
Gaze_Mapping_Plugin,
)
from eye_movement import Eye_Movement_Detector_Real_Time
from recorder import Recorder
from display_recent_gaze import Display_Recent_Gaze
from time_sync import Time_Sync
from pupil_remote import Pupil_Remote
from pupil_groups import Pupil_Groups
from surface_tracker import Surface_Tracker_Online
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
from frame_publisher import Frame_Publisher
from blink_detection import Blink_Detection
from video_capture import (
source_classes,
manager_classes,
Base_Manager,
Base_Source,
)
from pupil_data_relay import Pupil_Data_Relay
from remote_recorder import Remote_Recorder
from audio_capture import Audio_Capture
from accuracy_visualizer import Accuracy_Visualizer
from system_graphs import System_Graphs
from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
from hololens_relay import Hololens_Relay
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (30, 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (8, 90)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
icon_bar_width = 50
window_size = None
camera_render_size = None
hdpi_factor = 1.0
# g_pool holds variables for this process they are accessible to all plugins
g_pool = SimpleNamespace()
g_pool.app = "capture"
g_pool.process = "world"
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eye_procs_alive = eye_procs_alive
g_pool.preferred_remote_port = preferred_remote_port
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, "plugins"))
user_plugins = [
Audio_Capture,
Pupil_Groups,
Frame_Publisher,
Pupil_Remote,
Time_Sync,
Surface_Tracker_Online,
Annotation_Capture,
Log_History,
Eye_Movement_Detector_Real_Time,
Blink_Detection,
Remote_Recorder,
Accuracy_Visualizer,
Camera_Intrinsics_Estimation,
Hololens_Relay,
]
if platform.system() != "Windows":
# Head pose tracking is currently not available on Windows
from head_pose_tracker.online_head_pose_tracker import (
Online_Head_Pose_Tracker, )
user_plugins.append(Online_Head_Pose_Tracker)
system_plugins = ([
Log_Display,
Display_Recent_Gaze,
Recorder,
Pupil_Data_Relay,
Plugin_Manager,
System_Graphs,
] + manager_classes + source_classes)
plugins = (system_plugins + user_plugins + runtime_plugins +
calibration_plugins + gaze_mapping_plugins)
user_plugins += [
p for p in runtime_plugins if not isinstance(
p,
(
Base_Manager,
Base_Source,
System_Plugin_Base,
Calibration_Plugin,
Gaze_Mapping_Plugin,
),
)
]
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
default_capture_settings = {
"preferred_names": [
"Pupil Cam1 ID2",
"Logitech Camera",
"(046d:081d)",
"C510",
"B525",
"C525",
"C615",
"C920",
"C930e",
],
"frame_size": (1280, 720),
"frame_rate":
30,
}
default_plugins = [
("UVC_Source", default_capture_settings),
("Pupil_Data_Relay", {}),
("UVC_Manager", {}),
("Log_Display", {}),
("Dummy_Gaze_Mapper", {}),
("Display_Recent_Gaze", {}),
("Screen_Marker_Calibration", {}),
("Recorder", {}),
("Pupil_Remote", {}),
("Accuracy_Visualizer", {}),
("Plugin_Manager", {}),
("System_Graphs", {}),
]
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
if w == 0 or h == 0:
return
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
# call `on_drop` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(p.on_drop(paths) for p in g_pool.plugins)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, "user_settings_world"))
if VersionFormat(session_settings.get("version",
"0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.min_calibration_confidence = session_settings.get(
"min_calibration_confidence", 0.8)
g_pool.detection_mapping_mode = session_settings.get(
"detection_mapping_mode", "3d")
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
g_pool.capture = None
audio.audio_mode = session_settings.get("audio_mode",
audio.default_audio_mode)
def handle_notifications(noti):
subject = noti["subject"]
if subject == "set_detection_mapping_mode":
if noti["mode"] == "2d":
if ("Vector_Gaze_Mapper"
in g_pool.active_gaze_mapping_plugin.class_name):
logger.warning(
"The gaze mapper is not supported in 2d mode. Please recalibrate."
)
g_pool.plugins.add(
g_pool.plugin_by_name["Dummy_Gaze_Mapper"])
g_pool.detection_mapping_mode = noti["mode"]
elif subject == "start_plugin":
g_pool.plugins.add(g_pool.plugin_by_name[noti["name"]],
args=noti.get("args", {}))
elif subject == "stop_plugin":
for p in g_pool.plugins:
if p.class_name == noti["name"]:
p.alive = False
g_pool.plugins.clean()
elif subject == "eye_process.started":
noti = {
"subject": "set_detection_mapping_mode",
"mode": g_pool.detection_mapping_mode,
}
ipc_pub.notify(noti)
elif subject == "set_min_calibration_confidence":
g_pool.min_calibration_confidence = noti["value"]
elif subject.startswith("meta.should_doc"):
ipc_pub.notify({
"subject": "meta.doc",
"actor": g_pool.app,
"doc": world.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
"subject": "meta.doc",
"actor": p.class_name,
"doc": p.on_notify.__doc__,
})
elif subject == "world_process.adapt_window_size":
set_window_size()
width, height = session_settings.get("window_size",
(1280 + icon_bar_width, 720))
# window and gl setup
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height,
"Pupil Capture - World")
window_pos = session_settings.get("window_position",
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = (
camera_render_size[0] +
int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1],
)
logger.warning(icon_bar_width * g_pool.gui_user_scale *
hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
def reset_restart():
logger.warning("Resetting all settings and restarting Capture.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({"subject": "clear_settings_process.should_start"})
ipc_pub.notify({
"subject": "world_process.should_start",
"delay": 2.0
})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is opened, the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-400, 0),
size=(-icon_bar_width, 0),
header_pos="left")
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos="hidden")
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
(120, -100))
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.6, 0.8, 1.0, 1.2, 1.4],
label="Interface size",
))
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
on_resize(main_window, f_width, f_height)
general_settings.append(ui.Button("Reset window size",
set_window_size))
general_settings.append(
ui.Selector("audio_mode", audio, selection=audio.audio_modes))
general_settings.append(
ui.Selector(
"detection_mapping_mode",
g_pool,
label="detection & mapping mode",
setter=set_detection_mapping_mode,
selection=["disabled", "2d", "3d"],
))
general_settings.append(
ui.Switch(
"eye0_process",
label="Detect eye 0",
setter=lambda alive: start_stop_eye(0, alive),
getter=lambda: eye_procs_alive[0].value,
))
general_settings.append(
ui.Switch(
"eye1_process",
label="Detect eye 1",
setter=lambda alive: start_stop_eye(1, alive),
getter=lambda: eye_procs_alive[1].value,
))
general_settings.append(
ui.Info_Text("Capture Version: {}".format(g_pool.version)))
general_settings.append(
ui.Button("Restart with default settings", reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
# plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(
g_pool, session_settings.get("loaded_plugins", default_plugins))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
toggle_general_settings(True)
# now that we have a proper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
# trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
if session_settings.get("eye1_process_alive", True):
launch_eye_process(1, delay=0.6)
if session_settings.get("eye0_process_alive", True):
launch_eye_process(0, delay=0.3)
ipc_pub.notify({"subject": "world_process.started"})
logger.warning("Process started.")
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# a dictionary that allows plugins to post and read events
events = {}
# report time between now and the last loop interation
events["dt"] = get_dt()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# "blacklisted" events that were already sent
del events["pupil"]
del events["gaze"]
# delete if exists. More expensive than del, so only use it when key might not exist
events.pop("annotation", None)
# send new events to ipc:
if "frame" in events:
del events["frame"] # send explicitly with frame publisher
if "depth_frame" in events:
del events["depth_frame"]
if "audio_packets" in events:
del events["audio_packets"]
del events["dt"] # no need to send this
for data in events.values():
assert isinstance(data, (list, tuple))
for d in data:
ipc_pub.send(d)
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
glfw.glfwPollEvents()
if window_should_update() and gl_utils.is_window_visible(
main_window):
gl_utils.glViewport(0, 0, *camera_render_size)
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(
main_window).decode()
except AttributeError: # clipboard is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window,
user_input.clipboard.encode())
for button, action, mods in user_input.buttons:
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
# call `on_click` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(
p.on_click(pos, button, action)
for p in g_pool.plugins)
for key, scancode, action, mods in user_input.keys:
# call `on_key` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(
p.on_key(key, scancode, action, mods)
for p in g_pool.plugins)
for char_ in user_input.chars:
# call `char_` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(p.on_char(char_) for p in g_pool.plugins)
glfw.glfwSwapBuffers(main_window)
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["window_position"] = glfw.glfwGetWindowPos(
main_window)
session_settings["version"] = str(g_pool.version)
session_settings["eye0_process_alive"] = eye_procs_alive[0].value
session_settings["eye1_process_alive"] = eye_procs_alive[1].value
session_settings[
"min_calibration_confidence"] = g_pool.min_calibration_confidence
session_settings[
"detection_mapping_mode"] = g_pool.detection_mapping_mode
session_settings["audio_mode"] = audio.audio_mode
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
except:
import traceback
trace = traceback.format_exc()
logger.error("Process Capture crashed with trace:\n{}".format(trace))
finally:
# shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({"subject": "world_process.stopped"})
sleep(1.0)
def eye(
timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None,
hide_ui=False,
):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
``start_eye_plugin``: Start plugins in eye process
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
if is_alive_flag.value:
# indicates eye process that this is a duplicated startup
logger.warning("Aborting redundant eye process startup")
return
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
# general imports
import traceback
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible, glViewport
from ui_roi import UIRoi
# monitoring
import psutil
# Plug-ins
from plugin import Plugin_List
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, MPEG_Writer
from ndsi import H264Writer
from video_capture import source_classes, manager_classes
from background_helper import IPC_Logging_Task_Proxy
from pupil_detector_plugins import available_detector_plugins
from pupil_detector_plugins.manager import PupilDetectorManager
IPC_Logging_Task_Proxy.push_url = ipc_push_url
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
# NOTE: Interrupt is handled in world/service/player which are responsible for
# shutting down the eye process properly
signal.signal(signal.SIGINT, interrupt_handler)
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id + 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (600, 90 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
icon_bar_width = 50
window_size = None
camera_render_size = None
hdpi_factor = 1.0
# g_pool holds variables for this process
g_pool = SimpleNamespace()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = "capture"
g_pool.eye_id = eye_id
g_pool.process = f"eye{eye_id}"
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
default_detector_cls, available_detectors = available_detector_plugins(
)
plugins = (manager_classes + source_classes + available_detectors +
[PupilDetectorManager])
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
preferred_names = [
f"Pupil Cam3 ID{eye_id}",
f"Pupil Cam2 ID{eye_id}",
f"Pupil Cam1 ID{eye_id}",
]
if eye_id == 0:
preferred_names += ["HD-6000"]
default_capture_settings = (
"UVC_Source",
{
"preferred_names": preferred_names,
"frame_size": (320, 240),
"frame_rate": 120,
},
)
default_plugins = [
# TODO: extend with plugins
default_capture_settings,
("UVC_Manager", {}),
# Detector needs to be loaded first to set `g_pool.pupil_detector`
(default_detector_cls.__name__, {}),
("PupilDetectorManager", {}),
]
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x *= hdpi_factor
y *= hdpi_factor
g_pool.gui.update_mouse(x, y)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, g_pool.capture.frame_size)
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
for plugin in g_pool.plugins:
if plugin.on_drop(paths):
break
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir,
"user_settings_eye{}".format(eye_id)))
if VersionFormat(session_settings.get("version",
"0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.flip = session_settings.get("flip", False)
g_pool.display_mode = session_settings.get("display_mode",
"camera_image")
g_pool.display_mode_info_text = {
"camera_image":
"Raw eye camera image. This uses the least amount of CPU power",
"roi":
"Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
"algorithm":
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
}
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# Initialize glfw
glfw.glfwInit()
if hide_ui:
glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window
title = "Pupil Capture - eye {}".format(eye_id)
width, height = session_settings.get("window_size",
(640 + icon_bar_width, 480))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get("window_position",
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(
np.ones((1, 1), dtype=np.uint8) + 125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos="left")
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos="hidden")
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2],
label="Interface Size",
))
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width *= 2
f_height *= 2
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
def uroi_on_mouse_button(button, action, mods):
if g_pool.display_mode == "roi":
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
x, y = glfw.glfwGetCursorPos(main_window)
# pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
x *= hdpi_factor
y *= hdpi_factor
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(
pos,
g_pool.capture.frame_size) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,
g_pool.u_r.handle_size,
g_pool.u_r.handle_size):
# if the roi interacts we dont want
# the gui to interact as well
return
general_settings.append(ui.Button("Reset window size",
set_window_size))
general_settings.append(
ui.Switch("flip", g_pool, label="Flip image display"))
general_settings.append(
ui.Selector(
"display_mode",
g_pool,
setter=set_display_mode_info,
selection=["camera_image", "roi", "algorithm"],
labels=["Camera Image", "ROI", "Algorithm"],
label="Mode",
))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
toggle_general_settings(False)
plugins_to_load = session_settings.get("loaded_plugins",
default_plugins)
if overwrite_cap_settings:
# Ensure that overwrite_cap_settings takes preference over source plugins
# with incorrect settings that were loaded from session settings.
plugins_to_load.append(overwrite_cap_settings)
g_pool.plugins = Plugin_List(g_pool, plugins_to_load)
g_pool.writer = None
g_pool.u_r = UIRoi(
(g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get("roi")
if roi_user_settings and tuple(
roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 50)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 50)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = "jpeg"
frame_publish_format_recent_warning = False
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning("Process started.")
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification["subject"]
if subject.startswith("eye_process.should_stop"):
if notification["eye_id"] == eye_id:
break
elif subject == "recording.started":
if notification["record_eye"] and g_pool.capture.online:
record_path = notification["rec_path"]
raw_mode = notification["compression"]
start_time_synced = notification["start_time_synced"]
logger.info(
"Will save eye video to: {}".format(record_path))
video_path = os.path.join(record_path,
"eye{}.mp4".format(eye_id))
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(
video_path, start_time_synced)
elif hasattr(g_pool.capture._recent_frame,
"h264_buffer"):
g_pool.writer = H264Writer(
video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate,
)
else:
g_pool.writer = MPEG_Writer(
video_path, start_time_synced)
elif subject == "recording.stopped":
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
elif subject.startswith("meta.should_doc"):
ipc_socket.notify({
"subject": "meta.doc",
"actor": "eye{}".format(eye_id),
"doc": eye.__doc__,
})
elif subject.startswith("frame_publishing.started"):
should_publish_frames = True
frame_publish_format = notification.get("format", "jpeg")
elif subject.startswith("frame_publishing.stopped"):
should_publish_frames = False
frame_publish_format = "jpeg"
elif (subject.startswith("start_eye_plugin")
and notification["target"] == g_pool.process):
try:
g_pool.plugins.add(
g_pool.plugin_by_name[notification["name"]],
notification.get("args", {}),
)
except KeyError as err:
logger.error(f"Attempt to load unknown plugin: {err}")
for plugin in g_pool.plugins:
plugin.on_notify(notification)
event = {}
for plugin in g_pool.plugins:
plugin.recent_events(event)
frame = event.get("frame")
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
f_height,
f_width,
):
g_pool.pupil_detector.on_resolution_change(
(g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
g_pool.capture.frame_size,
)
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
assert data is not None
except (AttributeError, AssertionError, NameError):
if not frame_publish_format_recent_warning:
frame_publish_format_recent_warning = True
logger.warning(
'{}s are not compatible with format "{}"'.
format(type(frame), frame_publish_format))
else:
frame_publish_format_recent_warning = False
pupil_socket.send({
"topic":
"frame.eye.{}".format(eye_id),
"width":
frame.width,
"height":
frame.height,
"index":
frame.index,
"timestamp":
frame.timestamp,
"format":
frame_publish_format,
"__raw_data__": [data],
})
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1.0 / dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
result = event.get("pupil_detection_result", None)
if result is not None:
pupil_socket.send(result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
glViewport(0, 0, *camera_render_size)
for p in g_pool.plugins:
p.gl_display()
glViewport(0, 0, *camera_render_size)
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == "roi":
g_pool.u_r.draw_points(g_pool.gui.scale)
glViewport(0, 0, *window_size)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
try:
clipboard = glfw.glfwGetClipboardString(
main_window).decode()
except AttributeError: # clipboard is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(
main_window, user_input.clipboard.encode())
for button, action, mods in user_input.buttons:
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer.release()
g_pool.writer = None
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
# save session persistent settings
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["roi"] = g_pool.u_r.get()
session_settings["flip"] = g_pool.flip
session_settings["display_mode"] = g_pool.display_mode
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["version"] = str(g_pool.version)
if not hide_ui:
glfw.glfwRestoreWindow(
main_window) # need to do this for windows os
session_settings["window_position"] = glfw.glfwGetWindowPos(
main_window)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
for plugin in g_pool.plugins:
plugin.alive = False
g_pool.plugins.clean()
glfw.glfwDestroyWindow(main_window)
g_pool.gui.terminate()
glfw.glfwTerminate()
logger.info("Process shutting down.")
def eye(timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id):
#logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, make_coord_system_eye_camera_based, is_window_visible
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from uvc import get_time_monotonic, StreamError
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from av_writer import JPEG_Writer, AV_Writer
from video_capture import InitialisationError, StreamError, Fake_Source, EndofVideoFileError, source_classes, manager_classes
source_by_name = {src.class_name(): src for src in source_classes}
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D
}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos,
(frame.width, frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size + 40,
g_pool.u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version",
VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager', {}))
if eye_id == 0:
cap_src = [
"Pupil Cam1 ID0", "HD-6000", "Integrated Camera",
"HD USB Camera", "USB 2.0 Camera"
]
else:
cap_src = ["Pupil Cam1 ID1", "HD-6000", "Integrated Camera"]
# Initialize capture
default_settings = {
'source_class_name': 'UVC_Source',
'preferred_names': cap_src,
'frame_size': (640, 480),
'frame_rate': 90
}
settings = overwrite_cap_settings or session_settings.get(
'capture_settings', default_settings)
try:
cap = source_by_name[settings['source_class_name']](g_pool,
**settings)
except (KeyError, InitialisationError) as e:
if isinstance(e, KeyError):
logger.warning(
'Incompatible capture setting encountered. Falling back to fake source.'
)
cap = Fake_Source(g_pool, **settings)
g_pool.iconified = False
g_pool.capture = cap
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode',
'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."
}
g_pool.u_r = UIRoi(
(g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
writer = None
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,
pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s" % eye_id
width, height = session_settings.get('window_size',
g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
glfw.glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector(
'pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Detector_2D, Detector_3D],
labels=['C++ 2d detector', 'C++ 3d detector'],
label="Detection method")
general_settings.append(detector_selector)
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture.init_gui()
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__: c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](
g_pool, **manager_settings)
g_pool.capture_manager.init_gui()
def open_manager(manager_class):
g_pool.capture_manager.cleanup()
g_pool.capture_manager = manager_class(g_pool)
g_pool.capture_manager.init_gui()
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(
0,
ui.Selector('capture_manager',
g_pool,
setter=open_manager,
getter=lambda: g_pool.capture_manager.__class__,
selection=manager_classes,
labels=[b.gui_name for b in manager_classes],
label='Manager'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
should_publish_frames = False
frame_publish_format = 'jpeg'
#create a timer to control window update frequency
window_update_timer = timer(1 / 60.)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification['subject']
if subject == 'eye_process.should_stop':
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye']:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(
record_path, "eye%s_timestamps.npy" % eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path,
g_pool.capture.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = AV_Writer(video_path,
g_pool.capture.frame_rate)
timestamps = []
elif subject == 'recording.stopped':
if writer:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject': 'meta.doc',
'actor': 'eye%i' % eye_id,
'doc': eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format', 'jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
else:
g_pool.capture_manager.on_notify(notification)
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except StreamError as e:
logger.error("Error getting frame. Stopping eye process.")
logger.debug("Caught error: %s" % e)
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
g_pool.capture.seek_to_frame(0)
frame = g_pool.capture.get_frame()
g_pool.u_r = UIRoi((frame.height, frame.width))
g_pool.capture_manager.update(frame, {})
if should_publish_frames and frame.jpeg_buffer:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
pupil_socket.send(
'frame.eye.%s' % eye_id, {
'width': frame.width,
'height': frame.width,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s' % eye_id, result)
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based(
(frame.height, frame.width, 3), g_pool.flip)
g_pool.capture.gl_display()
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly(
(int(eye_ball['center'][0]),
int(eye_ball['center'][1])),
(int(eye_ball['axes'][0] / 2),
int(eye_ball['axes'][1] / 2)),
int(eye_ball['angle']), 0, 360, 8)
except ValueError as e:
pass
else:
draw_polyline(
pts, 2,
RGBA(0., .9, .1, result['model_confidence']))
if result['confidence'] > 0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
confidence = result[
'confidence'] * 0.7 #scale it a little
draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., confidence),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize(
) #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings[
'capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict(
)
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = g_pool.version
session_settings[
'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings[
'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_gui()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
logger.info("Process shutting down.")
def service(
timebase,
eye_procs_alive,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
preferred_remote_port,
hide_ui,
):
"""Maps pupil to gaze data, can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``start_plugin``: Starts given plugin with the given arguments
``eye_process.started``: Sets the detection method eye process
``service_process.should_stop``: Stops the service process
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``service_process.started``
``service_process.stopped``
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the service process each process also loads the other imports.
# This is not harmful but unnecessary.
# general imports
from time import sleep
import logging
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("pupil", ))
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
poller = zmq.Poller()
poller.register(pupil_sub.socket)
poller.register(notify_sub.socket)
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
def launch_eye_process(eye_id, delay=0):
n = {
"subject": "eye_process.should_start",
"eye_id": eye_id,
"delay": delay
}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {"subject": "eye_process.should_stop", "eye_id": eye_id}
ipc_pub.notify(n)
try:
# helpers/utils
from file_methods import Persistent_Dict
from methods import delta_t, get_system_info
from version_utils import VersionFormat
import audio
from uvc import get_time_monotonic
# trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from pupil_remote import Pupil_Remote
from pupil_groups import Pupil_Groups
from frame_publisher import Frame_Publisher
from blink_detection import Blink_Detection
from service_ui import Service_UI
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
process_was_interrupted = False
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
nonlocal process_was_interrupted
process_was_interrupted = True
signal.signal(signal.SIGINT, interrupt_handler)
logger.info("Application Version: {}".format(version))
logger.info("System Info: {}".format(get_system_info()))
# g_pool holds variables for this process they are accesible to all plugins
g_pool = SimpleNamespace()
g_pool.app = "service"
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.get_now = get_time_monotonic
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eye_procs_alive = eye_procs_alive
g_pool.timebase = timebase
g_pool.preferred_remote_port = preferred_remote_port
g_pool.hide_ui = hide_ui
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
# manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, "plugins"))
user_launchable_plugins = [
Service_UI,
Pupil_Groups,
Pupil_Remote,
Frame_Publisher,
Blink_Detection,
] + runtime_plugins
plugin_by_index = (runtime_plugins + calibration_plugins +
gaze_mapping_plugins + user_launchable_plugins)
name_by_index = [
pupil_datum.__name__ for pupil_datum in plugin_by_index
]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [
("Service_UI", {}),
("Dummy_Gaze_Mapper", {}),
("HMD_Calibration", {}),
("Pupil_Remote", {}),
]
g_pool.plugin_by_name = plugin_by_name
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, "user_settings_service"))
if session_settings.get("version",
VersionFormat("0.0")) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
g_pool.min_calibration_confidence = session_settings.get(
"min_calibration_confidence", 0.8)
g_pool.detection_mapping_mode = session_settings.get(
"detection_mapping_mode", "2d")
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
audio.audio_mode = session_settings.get("audio_mode",
audio.default_audio_mode)
ipc_pub.notify({"subject": "service_process.started"})
logger.warning("Process started.")
g_pool.service_should_run = True
# plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(
g_pool, session_settings.get("loaded_plugins", default_plugins))
# NOTE: The Pupil_Remote plugin fails to load when the port is already in use
# and will set this variable to false. Then we should not even start the eye
# processes. Otherwise we would have to wait for their initialization before
# attempting cleanup in Service.
if g_pool.service_should_run:
if session_settings.get("eye1_process_alive", True):
launch_eye_process(1, delay=0.3)
if session_settings.get("eye0_process_alive", True):
launch_eye_process(0, delay=0.0)
def handle_notifications(n):
subject = n["subject"]
if subject == "set_detection_mapping_mode":
if n["mode"] == "2d":
if ("Vector_Gaze_Mapper"
in g_pool.active_gaze_mapping_plugin.class_name):
logger.warning(
"The gaze mapper is not supported in 2d mode. Please recalibrate."
)
g_pool.plugins.add(plugin_by_name["Dummy_Gaze_Mapper"])
g_pool.detection_mapping_mode = n["mode"]
elif subject == "start_plugin":
g_pool.plugins.add(plugin_by_name[n["name"]],
args=n.get("args", {}))
elif subject == "eye_process.started":
n = {
"subject": "set_detection_mapping_mode",
"mode": g_pool.detection_mapping_mode,
}
ipc_pub.notify(n)
elif subject == "service_process.should_stop":
g_pool.service_should_run = False
elif subject.startswith("meta.should_doc"):
ipc_pub.notify({
"subject": "meta.doc",
"actor": g_pool.app,
"doc": service.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
"subject": "meta.doc",
"actor": p.class_name,
"doc": p.on_notify.__doc__,
})
# initiate ui update loop
ipc_pub.notify({
"subject": "service_process.ui.should_update",
"initial_delay": 1 / 40
})
# Event loop
while g_pool.service_should_run and not process_was_interrupted:
socks = dict(poller.poll())
if pupil_sub.socket in socks:
topic, pupil_datum = pupil_sub.recv()
new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(
pupil_datum)
for gaze_datum in new_gaze_data:
gaze_pub.send(gaze_datum)
events = {}
events["gaze"] = new_gaze_data
events["pupil"] = [pupil_datum]
for plugin in g_pool.plugins:
plugin.recent_events(events=events)
if notify_sub.socket in socks:
topic, n = notify_sub.recv()
handle_notifications(n)
for plugin in g_pool.plugins:
plugin.on_notify(n)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["version"] = str(g_pool.version)
session_settings["eye0_process_alive"] = eye_procs_alive[0].value
session_settings["eye1_process_alive"] = eye_procs_alive[1].value
session_settings[
"min_calibration_confidence"] = g_pool.min_calibration_confidence
session_settings[
"detection_mapping_mode"] = g_pool.detection_mapping_mode
session_settings["audio_mode"] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for pupil_datum in g_pool.plugins:
pupil_datum.alive = False
g_pool.plugins.clean()
except Exception:
import traceback
trace = traceback.format_exc()
logger.error("Process Service crashed with trace:\n{}".format(trace))
finally:
# shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({"subject": "service_process.stopped"})
# shut down launcher
n = {"subject": "launcher_process.should_stop"}
ipc_pub.notify(n)
sleep(1.0)
def eye(
timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None,
hide_ui=False,
debug=False,
pub_socket_hwm=None,
parent_application="capture",
):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
``start_eye_plugin``: Start plugins in eye process
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url, pub_socket_hwm)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
if is_alive_flag.value:
# indicates eye process that this is a duplicated startup
logger.warning("Aborting redundant eye process startup")
return
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
# general imports
import traceback
import numpy as np
import cv2
from OpenGL.GL import GL_COLOR_BUFFER_BIT
# display
import glfw
from gl_utils import GLFWErrorReporting
GLFWErrorReporting.set_default()
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
import gl_utils
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible, glViewport
# monitoring
import psutil
# Plug-ins
from plugin import Plugin_List
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import parse_version
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, MPEG_Writer, NonMonotonicTimestampError
from ndsi import H264Writer
from video_capture import source_classes, manager_classes
from roi import Roi
from background_helper import IPC_Logging_Task_Proxy
from pupil_detector_plugins import available_detector_plugins, EVENT_KEY
IPC_Logging_Task_Proxy.push_url = ipc_push_url
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
# NOTE: Interrupt is handled in world/service/player which are responsible for
# shutting down the eye process properly
signal.signal(signal.SIGINT, interrupt_handler)
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id + 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (600, 90 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
icon_bar_width = 50
window_size = None
content_scale = 1.0
# g_pool holds variables for this process
g_pool = SimpleNamespace()
# make some constants avaiable
g_pool.debug = debug
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = parent_application
g_pool.eye_id = eye_id
g_pool.process = f"eye{eye_id}"
g_pool.timebase = timebase
g_pool.camera_render_size = None
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_socket
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
def load_runtime_pupil_detection_plugins():
from plugin import import_runtime_plugins
from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin
plugins_path = os.path.join(g_pool.user_dir, "plugins")
for plugin in import_runtime_plugins(plugins_path):
if not isinstance(plugin, type):
continue
if not issubclass(plugin, PupilDetectorPlugin):
continue
if plugin is PupilDetectorPlugin:
continue
yield plugin
available_detectors = available_detector_plugins()
runtime_detectors = list(load_runtime_pupil_detection_plugins())
plugins = (
manager_classes
+ source_classes
+ available_detectors
+ runtime_detectors
+ [Roi]
)
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
preferred_names = [
f"Pupil Cam3 ID{eye_id}",
f"Pupil Cam2 ID{eye_id}",
f"Pupil Cam1 ID{eye_id}",
]
if eye_id == 0:
preferred_names += ["HD-6000"]
default_capture_name = "UVC_Source"
default_capture_settings = {
"preferred_names": preferred_names,
"frame_size": (192, 192),
"frame_rate": 120,
}
default_plugins = [
# TODO: extend with plugins
(default_capture_name, default_capture_settings),
("UVC_Manager", {}),
*[(p.__name__, {}) for p in available_detectors],
("NDSI_Manager", {}),
("HMD_Streaming_Manager", {}),
("File_Manager", {}),
("Roi", {}),
]
def consume_events_and_render_buffer():
glfw.make_context_current(main_window)
clear_gl_screen()
if all(c > 0 for c in g_pool.camera_render_size):
glViewport(0, 0, *g_pool.camera_render_size)
for p in g_pool.plugins:
p.gl_display()
glViewport(0, 0, *window_size)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
try:
clipboard = glfw.get_clipboard_string(main_window).decode()
except (AttributeError, glfw.GLFWError):
# clipboard is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.set_clipboard_string(main_window, user_input.clipboard)
for button, action, mods in user_input.buttons:
x, y = glfw.get_cursor_pos(main_window)
pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
main_window, x, y, cached_scale=None
)
pos = normalize(pos, g_pool.camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
# update screen
glfw.swap_buffers(main_window)
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal content_scale
is_minimized = bool(glfw.get_window_attrib(window, glfw.ICONIFIED))
if is_minimized:
return
# Always clear buffers on resize to make sure that there are no overlapping
# artifacts from previous frames.
gl_utils.glClear(GL_COLOR_BUFFER_BIT)
gl_utils.glClearColor(0, 0, 0, 1)
active_window = glfw.get_current_context()
glfw.make_context_current(window)
content_scale = gl_utils.get_content_scale(window)
framebuffer_scale = gl_utils.get_framebuffer_scale(window)
g_pool.gui.scale = content_scale
window_size = w, h
g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = content_scale
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.make_context_current(active_window)
# Minimum window size required, otherwise parts of the UI can cause openGL
# issues with permanent effects. Depends on the content scale, which can
# potentially be dynamically modified, so we re-adjust the size limits every
# time here.
min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale)
glfw.set_window_size_limits(
window,
min_size,
min_size,
glfw.DONT_CARE,
glfw.DONT_CARE,
)
# Needed, to update the window buffer while resizing
consume_events_and_render_buffer()
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
window, x, y, cached_scale=None
)
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, paths):
for plugin in g_pool.plugins:
if plugin.on_drop(paths):
break
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
)
if parse_version(session_settings.get("version", "0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
camera_is_physically_flipped = eye_id == 0
g_pool.iconified = False
g_pool.capture = None
g_pool.flip = session_settings.get("flip", camera_is_physically_flipped)
g_pool.display_mode = session_settings.get("display_mode", "camera_image")
g_pool.display_mode_info_text = {
"camera_image": "Raw eye camera image. This uses the least amount of CPU power",
"roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
"algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
}
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# Initialize glfw
glfw.init()
glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
if hide_ui:
glfw.window_hint(glfw.VISIBLE, 0) # hide window
title = "Pupil Capture - eye {}".format(eye_id)
# Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions.
# As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3
# cameras, we adjust the default eye window size to a 1:1 content aspect ratio.
# The size of 500 was chosen s.t. the menu still fits.
default_window_size = 500 + icon_bar_width, 500
width, height = session_settings.get("window_size", default_window_size)
main_window = glfw.create_window(width, height, title, None, None)
window_position_manager = gl_utils.WindowPositionManager()
window_pos = window_position_manager.new_window_position(
window=main_window,
default_position=window_position_default,
previous_position=session_settings.get("window_position", None),
)
glfw.set_window_pos(main_window, window_pos[0], window_pos[1])
glfw.make_context_current(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.menubar = ui.Scrolling_Menu(
"Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
)
g_pool.iconbar = ui.Scrolling_Menu(
"Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
)
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
def set_window_size():
# Get current capture frame size
f_width, f_height = g_pool.capture.frame_size
# Eye camera resolutions are too small to be used as default window sizes.
# We use double their size instead.
frame_scale_factor = 2
f_width *= frame_scale_factor
f_height *= frame_scale_factor
# Get current display scale factor
content_scale = gl_utils.get_content_scale(main_window)
framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
display_scale_factor = content_scale / framebuffer_scale
# Scale the capture frame size by display scale factor
f_width *= display_scale_factor
f_height *= display_scale_factor
# Increas the width to account for the added scaled icon bar width
f_width += icon_bar_width * display_scale_factor
# Set the newly calculated size (scaled capture frame size + scaled icon bar width)
glfw.set_window_size(main_window, int(f_width), int(f_height))
general_settings.append(ui.Button("Reset window size", set_window_size))
general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
general_settings.append(
ui.Selector(
"display_mode",
g_pool,
setter=set_display_mode_info,
selection=["camera_image", "roi", "algorithm"],
labels=["Camera Image", "ROI", "Algorithm"],
label="Mode",
)
)
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode]
)
general_settings.append(g_pool.display_mode_info)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
plugins_to_load = session_settings.get("loaded_plugins", default_plugins)
if overwrite_cap_settings:
# Ensure that overwrite_cap_settings takes preference over source plugins
# with incorrect settings that were loaded from session settings.
plugins_to_load.append(overwrite_cap_settings)
# Add runtime plugins to the list of plugins to load with default arguments,
# if not already restored from session settings
plugins_to_load_names = set(name for name, _ in plugins_to_load)
for runtime_detector in runtime_detectors:
runtime_name = runtime_detector.__name__
if runtime_name not in plugins_to_load_names:
plugins_to_load.append((runtime_name, {}))
g_pool.plugins = Plugin_List(g_pool, plugins_to_load)
if not g_pool.capture:
# Make sure we always have a capture running. Important if there was no
# capture stored in session settings.
g_pool.plugins.add(
g_pool.plugin_by_name[default_capture_name], default_capture_settings
)
toggle_general_settings(True)
g_pool.writer = None
g_pool.rec_path = None
# Register callbacks main_window
glfw.set_framebuffer_size_callback(main_window, on_resize)
glfw.set_window_iconify_callback(main_window, on_iconify)
glfw.set_key_callback(main_window, on_window_key)
glfw.set_char_callback(main_window, on_window_char)
glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
glfw.set_cursor_pos_callback(main_window, on_pos)
glfw.set_scroll_callback(main_window, on_scroll)
glfw.set_drop_callback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# If previously selected plugin was not loaded this time, we will have an
# expanded menubar without any menu selected. We need to ensure the menubar is
# collapsed in this case.
if all(submenu.collapsed for submenu in g_pool.menubar.elements):
g_pool.menubar.collapsed = True
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 50)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 50)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.get_framebuffer_size(main_window))
should_publish_frames = False
frame_publish_format = "jpeg"
frame_publish_format_recent_warning = False
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning("Process started.")
frame = None
if platform.system() == "Darwin":
# On macOS, calls to glfw.swap_buffers() deliberately take longer in case of
# occluded windows, based on the swap interval value. This causes an FPS drop
# and leads to problems when recording. To side-step this behaviour, the swap
# interval is set to zero.
#
# Read more about window occlusion on macOS here:
# https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html
glfw.swap_interval(0)
# Event loop
window_should_close = False
while not window_should_close:
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification["subject"]
if subject.startswith("eye_process.should_stop"):
if notification["eye_id"] == eye_id:
break
elif subject == "recording.started":
if notification["record_eye"] and g_pool.capture.online:
g_pool.rec_path = notification["rec_path"]
raw_mode = notification["compression"]
start_time_synced = notification["start_time_synced"]
logger.info(f"Will save eye video to: {g_pool.rec_path}")
video_path = os.path.join(
g_pool.rec_path, "eye{}.mp4".format(eye_id)
)
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(video_path, start_time_synced)
elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
g_pool.writer = H264Writer(
video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate,
)
else:
g_pool.writer = MPEG_Writer(video_path, start_time_synced)
elif subject == "recording.stopped":
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
else:
# TODO: wrap recording logic into plugin
g_pool.capture.intrinsics.save(
g_pool.rec_path, custom_name=f"eye{eye_id}"
)
finally:
g_pool.writer = None
elif subject.startswith("meta.should_doc"):
ipc_socket.notify(
{
"subject": "meta.doc",
"actor": "eye{}".format(eye_id),
"doc": eye.__doc__,
}
)
elif subject.startswith("frame_publishing.started"):
should_publish_frames = True
frame_publish_format = notification.get("format", "jpeg")
elif subject.startswith("frame_publishing.stopped"):
should_publish_frames = False
frame_publish_format = "jpeg"
elif (
subject.startswith("start_eye_plugin")
and notification["target"] == g_pool.process
):
try:
g_pool.plugins.add(
g_pool.plugin_by_name[notification["name"]],
notification.get("args", {}),
)
except KeyError as err:
logger.error(f"Attempt to load unknown plugin: {err}")
elif (
subject.startswith("stop_eye_plugin")
and notification["target"] == g_pool.process
):
try:
plugin_to_stop = g_pool.plugin_by_name[notification["name"]]
except KeyError as err:
logger.error(f"Attempt to load unknown plugin: {err}")
else:
plugin_to_stop.alive = False
g_pool.plugins.clean()
for plugin in g_pool.plugins:
plugin.on_notify(notification)
event = {}
for plugin in g_pool.plugins:
plugin.recent_events(event)
frame = event.get("frame")
if frame:
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
assert data is not None
except (AttributeError, AssertionError, NameError):
if not frame_publish_format_recent_warning:
frame_publish_format_recent_warning = True
logger.warning(
'{}s are not compatible with format "{}"'.format(
type(frame), frame_publish_format
)
)
else:
frame_publish_format_recent_warning = False
pupil_socket.send(
{
"topic": "frame.eye.{}".format(eye_id),
"width": frame.width,
"height": frame.height,
"index": frame.index,
"timestamp": frame.timestamp,
"format": frame_publish_format,
"__raw_data__": [data],
}
)
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1.0 / dt)
except ZeroDivisionError:
pass
if g_pool.writer:
try:
g_pool.writer.write_video_frame(frame)
except NonMonotonicTimestampError as e:
logger.error(
"Recorder received non-monotonic timestamp!"
" Stopping the recording!"
)
logger.debug(str(e))
ipc_socket.notify({"subject": "recording.should_stop"})
ipc_socket.notify(
{"subject": "recording.should_stop", "remote_notify": "all"}
)
for result in event.get(EVENT_KEY, ()):
pupil_socket.send(result)
# GL drawing
if window_should_update():
cpu_graph.update()
if is_window_visible(main_window):
consume_events_and_render_buffer()
glfw.poll_events()
window_should_close = glfw.window_should_close(main_window)
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer.release()
g_pool.writer = None
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
# save session persistent settings
session_settings["flip"] = g_pool.flip
session_settings["display_mode"] = g_pool.display_mode
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["version"] = str(g_pool.version)
if not hide_ui:
glfw.restore_window(main_window) # need to do this for windows os
session_settings["window_position"] = glfw.get_window_pos(main_window)
session_window_size = glfw.get_window_size(main_window)
if 0 not in session_window_size:
f_width, f_height = session_window_size
if platform.system() in ("Windows", "Linux"):
# Store unscaled window size as the operating system will scale the
# windows appropriately during launch on Windows and Linux.
f_width, f_height = (
f_width / content_scale,
f_height / content_scale,
)
session_settings["window_size"] = int(f_width), int(f_height)
session_settings.close()
for plugin in g_pool.plugins:
plugin.alive = False
g_pool.plugins.clean()
glfw.destroy_window(main_window)
g_pool.gui.terminate()
glfw.terminate()
logger.info("Process shutting down.")
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version):
# general imports
from time import sleep
import logging
from glob import glob
from time import time, strftime, localtime
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
from tasklib.background.patches import IPCLoggingPatch
IPCLoggingPatch.ipc_push_url = ipc_push_url
# imports
from file_methods import Persistent_Dict, next_export_sub_dir
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture, RGBA
import gl_utils
# capture
from video_capture import File_Source
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
import player_methods as pm
from csv_utils import write_key_value_file
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
# from vis_scan_path import Vis_Scan_Path
from seek_control import Seek_Control
from offline_surface_tracker import Offline_Surface_Tracker
# from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producer.gaze_from_recording import GazeFromRecording
from gaze_producer.gaze_from_offline_calibration import (
GazeFromOfflineCalibration, )
from system_graphs import System_Graphs
from system_timelines import System_Timelines
from blink_detection import Offline_Blink_Detection
from audio_playback import Audio_Playback
from video_export.plugins.imotions_exporter import iMotions_Exporter
from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
from video_export.plugins.world_video_exporter import World_Video_Exporter
from video_capture import File_Source
from video_overlay.plugins import Video_Overlay, Eye_Overlay
assert VersionFormat(pyglui_version) >= VersionFormat(
"1.23"), "pyglui out of date, please upgrade to newest version"
runtime_plugins = import_runtime_plugins(
os.path.join(user_dir, "plugins"))
system_plugins = [
Log_Display,
Seek_Control,
Plugin_Manager,
System_Graphs,
System_Timelines,
Audio_Playback,
]
user_plugins = [
Vis_Circle,
Vis_Fixation,
Vis_Polyline,
Vis_Light_Points,
Vis_Cross,
Vis_Watermark,
Eye_Overlay,
Video_Overlay,
# Vis_Scan_Path,
Offline_Fixation_Detector,
Offline_Blink_Detection,
Offline_Surface_Tracker,
Raw_Data_Exporter,
Annotation_Player,
Log_History,
Pupil_From_Recording,
Offline_Pupil_Detection,
GazeFromRecording,
GazeFromOfflineCalibration,
World_Video_Exporter,
iMotions_Exporter,
Eye_Video_Exporter,
] + runtime_plugins
if platform.system() != "Windows":
# Head pose tracking is currently not available on Windows
from head_pose_tracker.offline_head_pose_tracker import (
Offline_Head_Pose_Tracker, )
user_plugins.append(Offline_Head_Pose_Tracker)
plugins = system_plugins + user_plugins
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal hdpi_factor
if w == 0 or h == 0:
return
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
g_pool.camera_render_size = w - int(
icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
for path in paths:
if pm.is_pupil_rec_dir(path):
_restart_with_recording(path)
return
# call `on_drop` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(p.on_drop(paths) for p in g_pool.plugins)
def _restart_with_recording(rec_dir):
logger.debug("Starting new session with '{}'".format(rec_dir))
ipc_pub.notify({
"subject": "player_drop_process.should_start",
"rec_dir": rec_dir
})
glfw.glfwSetWindowShouldClose(g_pool.main_window, True)
tick = delta_t()
def get_dt():
return next(tick)
meta_info = pm.load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info("Application Version: {}".format(app_version))
logger.info("System Info: {}".format(get_system_info()))
icon_bar_width = 50
window_size = None
hdpi_factor = 1.0
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = "player"
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in valid_ext
][0]
File_Source(
g_pool,
timing="external",
source_path=video_path,
buffered_decoding=True,
fill_gaps=True,
)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version",
"0.0")) != app_version:
logger.info(
"Session setting are a different version of this app. I will not use those."
)
session_settings.clear()
width, height = g_pool.capture.frame_size
width += icon_bar_width
width, height = session_settings.get("window_size", (width, height))
window_pos = session_settings.get("window_position",
window_position_default)
window_name = "Pupil Player: {} - {}".format(
meta_info["Recording Name"],
os.path.split(rec_dir)[-1])
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, window_name, None,
None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = (
g_pool.camera_render_size[0] +
int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1],
)
logger.warning(icon_bar_width * g_pool.gui_user_scale *
hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
# load pupil_positions, gaze_positions
g_pool.binocular = meta_info.get("Eye Mode",
"monocular") == "binocular"
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.0
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get(
"min_data_confidence", 0.6)
g_pool.min_calibration_confidence = session_settings.get(
"min_calibration_confidence", 0.8)
# populated by producers
g_pool.pupil_positions = pm.Bisector()
g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
g_pool.gaze_positions = pm.Bisector()
g_pool.fixations = pm.Affiliator()
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {"subject": "min_data_confidence_changed"}
notification["_notify_time_"] = time() + 0.8
g_pool.ipc_pub.notify(notification)
def do_export(_):
left_idx = g_pool.seek_control.trim_left
right_idx = g_pool.seek_control.trim_right
export_range = left_idx, right_idx + 1 # exclusive range.stop
export_ts_window = pm.exact_window(g_pool.timestamps,
(left_idx, right_idx))
export_dir = os.path.join(g_pool.rec_dir, "exports")
export_dir = next_export_sub_dir(export_dir)
os.makedirs(export_dir)
logger.info('Created export dir at "{}"'.format(export_dir))
export_info = {
"Player Software Version":
str(g_pool.version),
"Data Format Version":
meta_info["Data Format Version"],
"Export Date":
strftime("%d.%m.%Y", localtime()),
"Export Time":
strftime("%H:%M:%S", localtime()),
"Frame Index Range:":
g_pool.seek_control.get_frame_index_trim_range_string(),
"Relative Time Range":
g_pool.seek_control.get_rel_time_trim_range_string(),
"Absolute Time Range":
g_pool.seek_control.get_abs_time_trim_range_string(),
}
with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
write_key_value_file(csv, export_info)
notification = {
"subject": "should_export",
"range": export_range,
"ts_window": export_ts_window,
"export_dir": export_dir,
}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({"subject": "clear_settings_process.should_start"})
ipc_pub.notify({
"subject": "player_process.should_start",
"rec_dir": rec_dir,
"delay": 2.0,
})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos="left")
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos="hidden")
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
g_pool.user_timelines = ui.Timeline_Menu("User Timelines",
pos=(0.0, -150.0),
size=(0.0, 0.0),
header_pos="headline")
g_pool.user_timelines.color = RGBA(a=0.0)
g_pool.user_timelines.collapsed = True
# add container that constaints itself to the seekbar height
vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
vert_constr.append(g_pool.user_timelines)
g_pool.timelines.append(vert_constr)
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(ui.Button("Reset window size",
set_window_size))
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2] +
list(np.arange(1.5, 5.1, 0.5)),
label="Interface Size",
))
general_settings.append(
ui.Info_Text("Player Version: {}".format(g_pool.version)))
general_settings.append(
ui.Info_Text("Capture Version: {}".format(
meta_info["Capture Software Version"])))
general_settings.append(
ui.Info_Text("Data Format Version: {}".format(
meta_info["Data Format Version"])))
general_settings.append(
ui.Info_Text(
"High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
))
general_settings.append(
ui.Slider(
"min_data_confidence",
g_pool,
setter=set_data_confidence,
step=0.05,
min=0.0,
max=1.0,
label="Minimum data confidence",
))
general_settings.append(
ui.Button("Restart with default settings", reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
(100, -100))
g_pool.export_button = ui.Thumb(
"export",
label=chr(0xE2C5),
getter=lambda: False,
setter=do_export,
hotkey="e",
label_font="pupil_icons",
)
g_pool.quickbar.extend([g_pool.export_button])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [
("Plugin_Manager", {}),
("Seek_Control", {}),
("Log_Display", {}),
("Raw_Data_Exporter", {}),
("Vis_Polyline", {}),
("Vis_Circle", {}),
("System_Graphs", {}),
("System_Timelines", {}),
("World_Video_Exporter", {}),
("Pupil_From_Recording", {}),
("GazeFromRecording", {}),
("Audio_Playback", {}),
]
g_pool.plugins = Plugin_List(
g_pool, session_settings.get("loaded_plugins", default_plugins))
# Manually add g_pool.capture to the plugin list
g_pool.plugins._plugins.append(g_pool.capture)
g_pool.plugins._plugins.sort(key=lambda p: p.order)
g_pool.capture.init_ui()
general_settings.insert(
-1,
ui.Text_Input(
"rel_time_trim_section",
getter=g_pool.seek_control.get_rel_time_trim_range_string,
setter=g_pool.seek_control.set_rel_time_trim_range_string,
label="Relative time range to export",
),
)
general_settings.insert(
-1,
ui.Text_Input(
"frame_idx_trim_section",
getter=g_pool.seek_control.get_frame_index_trim_range_string,
setter=g_pool.seek_control.set_frame_index_trim_range_string,
label="Frame index range to export",
),
)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
toggle_general_settings(True)
g_pool.gui.configuration = session_settings.get("ui_config", {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n["subject"]
if subject == "start_plugin":
g_pool.plugins.add(g_pool.plugin_by_name[n["name"]],
args=n.get("args", {}))
elif subject.startswith("meta.should_doc"):
ipc_pub.notify({
"subject": "meta.doc",
"actor": g_pool.app,
"doc": player.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
"subject": "meta.doc",
"actor": p.class_name,
"doc": p.on_notify.__doc__,
})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
events = {}
# report time between now and the last loop interation
events["dt"] = get_dt()
# pupil and gaze positions are added by their respective producer plugins
events["pupil"] = []
events["gaze"] = []
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(
main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window,
user_input.clipboard.encode())
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
# call `on_click` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(
p.on_click(pos, button, action)
for p in g_pool.plugins)
for key, scancode, action, mods in user_input.keys:
# call `on_key` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(
p.on_key(key, scancode, action, mods)
for p in g_pool.plugins)
for char_ in user_input.chars:
# call `char_` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(p.on_char(char_) for p in g_pool.plugins)
# present frames at appropriate speed
g_pool.seek_control.wait(events["frame"].timestamp)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["min_data_confidence"] = g_pool.min_data_confidence
session_settings[
"min_calibration_confidence"] = g_pool.min_calibration_confidence
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["window_position"] = glfw.glfwGetWindowPos(
main_window)
session_settings["version"] = str(g_pool.version)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except:
import traceback
trace = traceback.format_exc()
logger.error("Process Player crashed with trace:\n{}".format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({"subject": "player_process.stopped"})
sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, app_version):
# general imports
import logging
import errno
from glob import glob
from copy import deepcopy
from time import time
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from file_methods import Persistent_Dict, load_object
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
import gl_utils
# capture
from video_capture import File_Source, EndofVideoFileError, FileSeekError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
# monitoring
import psutil
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_bar import Seek_Bar
from trim_marks import Trim_Marks
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
# from manual_gaze_correction import Manual_Gaze_Correction
from batch_exporter import Batch_Exporter
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
assert pyglui_version >= '1.7'
runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector,
Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher,
Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player,
Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection,
Gaze_From_Recording, Offline_Calibration] + runtime_plugins
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
if gl_utils.is_window_visible(window):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
# Initialize capture
cap = File_Source(g_pool, video_path)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if VersionFormat(session_settings.get("version", '0.0')) != app_version:
logger.info("Session setting are a different version of this app. I will not use those.")
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', window_position_default)
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
+ rec_dir.split(os.path.sep)[-1], None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() + 1)
except(FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 1)
except(FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if cap.get_frame_index() >= cap.get_frame_count()-5:
cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(*export_range))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.ipc_pub.notify(notification)
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560))
g_pool.main_menu.append(ui.Button('Reset window size',
lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
step=.05, min=0.0, max=1.0, label='Confidence threshold'))
g_pool.main_menu.append(ui.Info_Text('Open plugins'))
selector_label = "Select to load"
def append_selector(label, plugins):
plugins.sort(key=lambda p: p.__name__)
plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
g_pool.main_menu.append(ui.Selector(label,
selection=[selector_label] + plugins,
labels=[selector_label] + plugin_labels,
setter=open_plugin,
getter=lambda: selector_label))
base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base]
base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
launchable = user_launchable_plugins.copy()
for base_class, label in zip(base_plugins, base_labels):
member_plugins = []
for p in user_launchable_plugins:
if issubclass(p, base_class):
member_plugins.append(p)
launchable.remove(p)
append_selector(label, member_plugins)
# launchable only contains plugins that could not be assigned to any of the above categories
append_selector('Other', launchable)
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=chr(0xf04b),
setter=toggle_play,
hotkey=glfw.GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xf04e),
getter=lambda: False,
setter=next_frame,
hotkey=glfw.GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xf04a),
getter=lambda: False,
setter=prev_frame,
hotkey=glfw.GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=chr(0xf063),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}),
('Vis_Circle', {}), ('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = None
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(
plugin_by_name[n['name']], args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts and ts != t:
dt, ts = t-ts, t
fps_graph.add(1./dt)
else:
ts = new_frame.timestamp
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfw.glfwMakeContextCurrent(main_window)
gl_utils.make_coord_system_norm_based()
g_pool.image_tex.update_from_ndarray(frame.bgr)
g_pool.image_tex.draw()
gl_utils.make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
unused_elements = g_pool.gui.update()
for b in unused_elements.buttons:
button, action, mods = b
pos = glfw.glfwGetCursorPos(main_window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in unused_elements.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in unused_elements.chars:
for p in g_pool.plugins:
p.on_char(char_)
# present frames at appropriate speed
cap.wait(frame)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
import json
ctx = zmq.Context()
requester = ctx.socket(zmq.REQ)
# ip = '10.16.3.140'
ip = 'localhost'
port = 50020
requester.connect('tcp://%s:%s' % (ip, port))
requester.send_string('SUB_PORT')
sub_port = requester.recv_string()
TCP_IP = '0.0.0.0'
TCP_PORT = 5005
BUFFER_SIZE = 20 # Normally 1024, but we want fast response
ipc_sub_url = 'tcp://%s:%s' % (ip, sub_port)
print(ipc_sub_url)
pupil_sub = zmq_tools.Msg_Receiver(ctx, ipc_sub_url, topics=('gaze', ))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP, TCP_PORT))
s.listen(1)
while True:
try:
print('Socket established. Waiting for connection...')
conn, addr = s.accept()
print('Connection address:', addr)
while True:
topic, pupil_datum = pupil_sub.recv()
str_data = json.dumps(pupil_datum) + "\n"
print(topic, ':', str_data)
conn.send(str_data.encode())
except KeyboardInterrupt:
raise
def service(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, version):
"""Maps pupil to gaze data, can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``start_plugin``: Starts given plugin with the given arguments
``eye_process.started``: Sets the detection method eye process
``service_process.should_stop``: Stops the service process
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``service_process.started``
``service_process.stopped``
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the service process each process also loads the other imports.
# This is not harmful but unnecessary.
# general imports
from time import sleep
import logging
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('pupil', ))
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('notify', ))
poller = zmq.Poller()
poller.register(pupil_sub.socket)
poller.register(notify_sub.socket)
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
# helpers/utils
from file_methods import Persistent_Dict
from methods import delta_t, get_system_info
from version_utils import VersionFormat
import audio
from uvc import get_time_monotonic
# trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from pupil_remote import Pupil_Remote
from pupil_groups import Pupil_Groups
logger.info('Application Version: {}'.format(version))
logger.info('System Info: {}'.format(get_system_info()))
# g_pool holds variables for this process they are accesible to all plugins
g_pool = Global_Container()
g_pool.app = 'service'
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.get_now = get_time_monotonic
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eyes_are_alive = eyes_are_alive
g_pool.timebase = timebase
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
# manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, 'plugins'))
user_launchable_plugins = [Pupil_Groups, Pupil_Remote
] + runtime_plugins
plugin_by_index = runtime_plugins + calibration_plugins + gaze_mapping_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [('Dummy_Gaze_Mapper', {}), ('HMD_Calibration', {}),
('Pupil_Remote', {})]
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_service'))
if session_settings.get("version",
VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
g_pool.detection_mapping_mode = session_settings.get(
'detection_mapping_mode', '2d')
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
audio.audio_mode = session_settings.get('audio_mode',
audio.default_audio_mode)
# plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
def launch_eye_process(eye_id, delay=0):
n = {
'subject': 'eye_process.should_start',
'eye_id': eye_id,
'delay': delay
}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id}
ipc_pub.notify(n)
def handle_notifications(n):
subject = n['subject']
if subject == 'set_detection_mapping_mode':
if n['mode'] == '2d':
if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name:
logger.warning(
"The gaze mapper is not supported in 2d mode. Please recalibrate."
)
g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
g_pool.detection_mapping_mode = n['mode']
elif subject == 'start_plugin':
g_pool.plugins.add(plugin_by_name[n['name']],
args=n.get('args', {}))
elif subject == 'eye_process.started':
n = {
'subject': 'set_detection_mapping_mode',
'mode': g_pool.detection_mapping_mode
}
ipc_pub.notify(n)
elif subject == 'service_process.should_stop':
g_pool.service_should_run = False
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': g_pool.app,
'doc': service.__doc__
})
for p in g_pool.plugins:
if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify:
ipc_pub.notify({
'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__
})
if session_settings.get('eye1_process_alive', False):
launch_eye_process(1, delay=0.3)
if session_settings.get('eye0_process_alive', True):
launch_eye_process(0, delay=0.0)
ipc_pub.notify({'subject': 'service_process.started'})
logger.warning('Process started.')
g_pool.service_should_run = True
# Event loop
while g_pool.service_should_run:
socks = dict(poller.poll())
if pupil_sub.socket in socks:
t, p = pupil_sub.recv()
new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(
p)
for g in new_gaze_data:
gaze_pub.send('gaze', g)
events = {}
events['gaze_positions'] = new_gaze_data
events['pupil_positions'] = [p]
for plugin in g_pool.plugins:
plugin.recent_events(events=events)
if notify_sub.socket in socks:
t, n = notify_sub.recv()
handle_notifications(n)
for plugin in g_pool.plugins:
plugin.on_notify(n)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['version'] = g_pool.version
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings[
'detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
except:
import traceback
trace = traceback.format_exc()
logger.error('Process Service crashed with trace:\n{}'.format(trace))
finally:
# shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'service_process.stopped'})
# shut down launcher
n = {'subject': 'launcher_process.should_stop'}
ipc_pub.notify(n)
sleep(1.0)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, version):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``
``eye_process.started``
``start_plugin``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events
that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
# general imports
from time import sleep
import logging
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
# logger.setLevel(logging.DEBUG)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
def launch_eye_process(eye_id, delay=0):
n = {'subject': 'eye_process.should_start.{}'.format(eye_id),
'eye_id': eye_id, 'delay': delay}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {'subject': 'eye_process.should_stop.{}'.format(eye_id), 'eye_id': eye_id,'delay':0.2}
ipc_pub.notify(n)
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
ipc_pub.notify(n)
try:
# display
import glfw
from version_utils import VersionFormat
from pyglui import ui, cygl, __version__ as pyglui_version
assert VersionFormat(pyglui_version) >= VersionFormat('1.15'), 'pyglui out of date, please upgrade to newest version'
from pyglui.cygl.utils import Named_Texture
import gl_utils
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info, timer
from uvc import get_time_monotonic
logger.info('Application Version: {}'.format(version))
logger.info('System Info: {}'.format(get_system_info()))
import audio
# trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin, System_Plugin_Base, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from calibration_routines import calibration_plugins, gaze_mapping_plugins, Calibration_Plugin, Gaze_Mapping_Plugin
from fixation_detector import Fixation_Detector
from recorder import Recorder
from display_recent_gaze import Display_Recent_Gaze
from time_sync import Time_Sync
from pupil_remote import Pupil_Remote
from pupil_groups import Pupil_Groups
from surface_tracker import Surface_Tracker
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
from frame_publisher import Frame_Publisher
from blink_detection import Blink_Detection
from video_capture import source_classes, manager_classes,Base_Manager,Base_Source
from pupil_data_relay import Pupil_Data_Relay
from remote_recorder import Remote_Recorder
from audio_capture import Audio_Capture
from accuracy_visualizer import Accuracy_Visualizer
# from saccade_detector import Saccade_Detector
from system_graphs import System_Graphs
from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
from hololens_relay import Hololens_Relay
# UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (30, 30)
elif platform.system() == 'Windows':
scroll_factor = 10.0
window_position_default = (8, 90)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
icon_bar_width = 50
window_size = None
camera_render_size = None
hdpi_factor = 1.0
# g_pool holds variables for this process they are accessible to all plugins
g_pool = Global_Container()
g_pool.app = 'capture'
g_pool.process = 'world'
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eyes_are_alive = eyes_are_alive
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins'))
user_plugins = [Audio_Capture,
Pupil_Groups,
Frame_Publisher,
Pupil_Remote,
Time_Sync,
Surface_Tracker,
Annotation_Capture,
Log_History,
Fixation_Detector,
Blink_Detection,
Remote_Recorder,
Accuracy_Visualizer,
Camera_Intrinsics_Estimation,
Hololens_Relay]
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder, Pupil_Data_Relay, Plugin_Manager, System_Graphs] + manager_classes + source_classes
plugins = system_plugins + user_plugins + runtime_plugins + calibration_plugins + gaze_mapping_plugins
user_plugins += [p for p in runtime_plugins if not isinstance(p, (Base_Manager, Base_Source, System_Plugin_Base,
Calibration_Plugin, Gaze_Mapping_Plugin))]
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
default_capture_settings = {
'preferred_names': ["Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)",
"C510", "B525", "C525", "C615", "C920", "C930e"],
'frame_size': (1280, 720),
'frame_rate': 30
}
default_plugins = [("UVC_Source", default_capture_settings),
('Pupil_Data_Relay', {}),
('UVC_Manager', {}),
('Log_Display', {}),
('Dummy_Gaze_Mapper', {}),
('Display_Recent_Gaze', {}),
('Screen_Marker_Calibration', {}),
('Recorder', {}),
('Pupil_Remote', {}),
('Plugin_Manager', {}),
('System_Graphs', {})]
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
if w == 0 or h == 0:
return
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w,h
camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode('utf-8') for x in range(count)]
for p in g_pool.plugins:
p.on_drop(paths)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir, 'user_settings_world'))
if VersionFormat(session_settings.get("version", '0.0')) != g_pool.version:
logger.info("Session setting are from a different version of this app. I will not use those.")
session_settings.clear()
g_pool.min_calibration_confidence = session_settings.get('min_calibration_confidence', 0.8)
g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode', '3d')
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
g_pool.capture = None
audio.audio_mode = session_settings.get('audio_mode', audio.default_audio_mode)
def handle_notifications(n):
subject = n['subject']
if subject == 'set_detection_mapping_mode':
if n['mode'] == '2d':
if ("Vector_Gaze_Mapper" in
g_pool.active_gaze_mapping_plugin.class_name):
logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
g_pool.plugins.add(g_pool.plugin_by_name['Dummy_Gaze_Mapper'])
g_pool.detection_mapping_mode = n['mode']
elif subject == 'start_plugin':
g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {}))
elif subject == 'stop_plugin':
for p in g_pool.plugins:
if p.class_name == n['name']:
p.alive = False
g_pool.plugins.clean()
elif subject == 'eye_process.started':
n = {'subject': 'set_detection_mapping_mode',
'mode': g_pool.detection_mapping_mode}
ipc_pub.notify(n)
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({'subject': 'meta.doc',
'actor': g_pool.app,
'doc': world.__doc__})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__})
width, height = session_settings.get('window_size', (1280 + icon_bar_width, 720))
# window and gl setup
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World")
window_pos = session_settings.get('window_position', window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = camera_render_size[0] + \
int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), \
glfw.glfwGetFramebufferSize(main_window)[1]
logger.warning(icon_bar_width*g_pool.gui_user_scale*hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
def reset_restart():
logger.warning("Resetting all settings and restarting Capture.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
ipc_pub.notify({'subject': 'world_process.should_start', 'delay': 2.})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is opened, the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos='left')
g_pool.iconbar = ui.Scrolling_Menu("Icons",pos=(-icon_bar_width,0),size=(0,0),header_pos='hidden')
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
general_settings = ui.Growing_Menu('General',header_pos='headline')
general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.6, .8, 1., 1.2, 1.4], label='Interface size'))
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width*g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
general_settings.append(ui.Button('Reset window size', set_window_size))
general_settings.append(ui.Selector('audio_mode', audio, selection=audio.audio_modes))
general_settings.append(ui.Selector('detection_mapping_mode',
g_pool,
label='detection & mapping mode',
setter=set_detection_mapping_mode,
selection=['disabled', '2d', '3d']))
general_settings.append(ui.Switch('eye0_process',
label='Detect eye 0',
setter=lambda alive: start_stop_eye(0,alive),
getter=lambda: eyes_are_alive[0].value
))
general_settings.append(ui.Switch('eye1_process',
label='Detect eye 1',
setter=lambda alive: start_stop_eye(1,alive),
getter=lambda: eyes_are_alive[1].value
))
general_settings.append(ui.Info_Text('Capture Version: {}'.format(g_pool.version)))
general_settings.append(ui.Button('Restart with default settings', reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons')
icon.tooltip = 'General Settings'
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
# plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(g_pool, session_settings.get('loaded_plugins', default_plugins))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
toggle_general_settings(True)
# now that we have a proper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
# trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
if session_settings.get('eye1_process_alive', False):
launch_eye_process(1, delay=0.6)
if session_settings.get('eye0_process_alive', True):
launch_eye_process(0, delay=0.3)
ipc_pub.notify({'subject': 'world_process.started'})
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# a dictionary that allows plugins to post and read events
events = {}
# report time between now and the last loop interation
events['dt'] = get_dt()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# send new events to ipc:
del events['pupil_positions'] # already on the wire
del events['gaze_positions'] # sent earlier
if 'frame' in events:
del events['frame'] # send explicitly with frame publisher
if 'depth_frame' in events:
del events['depth_frame']
if 'audio_packets' in events:
del events['audio_packets']
del events['dt'] # no need to send this
for topic, data in events.items():
assert(isinstance(data, (list, tuple)))
for d in data:
ipc_pub.send(topic, d)
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
if window_should_update() and gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *camera_render_size)
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(main_window).decode()
except AttributeError: # clipboard is None, might happen on startup
clipboard = ''
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode())
for button, action, mods in user_input.buttons:
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in user_input.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in user_input.chars:
for p in g_pool.plugins:
p.on_char(char_)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['min_calibration_confidence'] = g_pool.min_calibration_confidence
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings['window_size'] = session_window_size
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
except:
import traceback
trace = traceback.format_exc()
logger.error('Process Capture crashed with trace:\n{}'.format(trace))
finally:
# shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'world_process.stopped'})
sleep(1.0)
def world(timebase, eyes_are_alive, ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, version, cap_src):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``
``eye_process.started``
``start_plugin``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
#general imports
from time import time, sleep
import numpy as np
import logging
import zmq
import zmq_tools
#zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('pupil', ))
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('notify', ))
#log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, glFlush
#check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
assert pyglui_version >= '0.8'
#monitoring
import psutil
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from version_utils import VersionFormat
import audio
from uvc import get_time_monotonic
#trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from recorder import Recorder
from show_calibration import Show_Calibration
from display_recent_gaze import Display_Recent_Gaze
from pupil_sync import Pupil_Sync
from pupil_remote import Pupil_Remote
from surface_tracker import Surface_Tracker
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
logger.info('Application Version: %s' % version)
logger.info('System Info: %s' % get_system_info())
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (0, 0)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (8, 31)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
#g_pool holds variables for this process they are accesible to all plugins
g_pool = Global_Container()
g_pool.app = 'capture'
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eyes_are_alive = eyes_are_alive
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
#manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, 'plugins'))
user_launchable_plugins = [
Show_Calibration, Pupil_Remote, Pupil_Sync, Surface_Tracker,
Annotation_Capture, Log_History
] + runtime_plugins
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
('Display_Recent_Gaze', {}),
('Screen_Marker_Calibration', {}), ('Recorder', {}),
('Pupil_Remote', {})]
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': (1280, 720), 'frame_rate': 30}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
g_pool.iconified = False
g_pool.capture = cap
g_pool.detection_mapping_mode = session_settings.get(
'detection_mapping_mode', '2d')
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
audio.audio_mode = session_settings.get('audio_mode',
audio.default_audio_mode)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def launch_eye_process(eye_id, delay=0):
n = {
'subject': 'eye_process.should_start.%s' % eye_id,
'eye_id': eye_id,
'delay': delay
}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {'subject': 'eye_process.should_stop', 'eye_id': eye_id}
ipc_pub.notify(n)
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
n = {'subject': 'set_detection_mapping_mode', 'mode': new_mode}
ipc_pub.notify(n)
def handle_notifications(n):
subject = n['subject']
if subject == 'set_detection_mapping_mode':
if n['mode'] == '2d':
if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name:
logger.warning(
"The gaze mapper is not supported in 2d mode. Please recalibrate."
)
g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
g_pool.detection_mapping_mode = n['mode']
elif subject == 'start_plugin':
g_pool.plugins.add(plugin_by_name[n['name']],
args=n.get('args', {}))
elif subject == 'eye_process.started':
n = {
'subject': 'set_detection_mapping_mode',
'mode': g_pool.detection_mapping_mode
}
ipc_pub.notify(n)
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': g_pool.app,
'doc': world.__doc__
})
for p in g_pool.plugins:
if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify:
ipc_pub.notify({
'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__
})
#window and gl setup
glfw.glfwInit()
width, height = session_settings.get('window_size', cap.frame_size)
main_window = glfw.glfwCreateWindow(width, height, "World")
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-350, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('audio_mode', audio, selection=audio.audio_modes))
general_settings.append(
ui.Selector('detection_mapping_mode',
g_pool,
label='detection & mapping mode',
setter=set_detection_mapping_mode,
selection=['2d', '3d']))
general_settings.append(
ui.Switch('eye0_process',
label='Detect eye 0',
setter=lambda alive: start_stop_eye(0, alive),
getter=lambda: eyes_are_alive[0].value))
general_settings.append(
ui.Switch('eye1_process',
label='Detect eye 1',
setter=lambda alive: start_stop_eye(1, alive),
getter=lambda: eyes_are_alive[1].value))
general_settings.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
general_settings.append(
ui.Info_Text('Capture Version: %s' % g_pool.version))
g_pool.sidebar.append(general_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.gui.append(g_pool.quickbar)
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(
0,
ui.Selector(
'active_calibration_plugin',
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection=calibration_plugins,
labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
setter=open_plugin,
label='Method'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
# refresh speed settings
glfw.glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil0_graph = graph.Bar_Graph(max_val=1.0)
pupil0_graph.pos = (260, 130)
pupil0_graph.update_rate = 5
pupil0_graph.label = "id0 conf: %0.2f"
pupil1_graph = graph.Bar_Graph(max_val=1.0)
pupil1_graph.pos = (380, 130)
pupil1_graph.update_rate = 5
pupil1_graph.label = "id1 conf: %0.2f"
pupil_graphs = pupil0_graph, pupil1_graph
if session_settings.get('eye1_process_alive', False):
launch_eye_process(1, delay=0.6)
if session_settings.get('eye0_process_alive', True):
launch_eye_process(0, delay=0.3)
ipc_pub.notify({'subject': 'world_process.started'})
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Starting Fake Capture.")
settings = g_pool.capture.settings
g_pool.capture.close()
g_pool.capture = autoCreateCapture(None, timebase=g_pool.timebase)
g_pool.capture.init_gui(g_pool.sidebar)
g_pool.capture.settings = settings
ipc_pub.notify({'subject': 'recording.should_stop'})
continue
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
recent_pupil_data = []
recent_gaze_data = []
new_notifications = []
while pupil_sub.new_data:
t, p = pupil_sub.recv()
pupil_graphs[p['id']].add(p['confidence'])
recent_pupil_data.append(p)
new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p)
for g in new_gaze_data:
gaze_pub.send('gaze', g)
recent_gaze_data += new_gaze_data
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
events['pupil_positions'] = recent_pupil_data
events['gaze_positions'] = recent_gaze_data
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
#send new events to ipc:
del events['pupil_positions'] #already on the wire
del events['gaze_positions'] #send earlier in this loop
del events['dt'] #no need to send this
for topic, data in events.iteritems():
for d in data:
ipc_pub.send(topic, d)
# render camera image
glfw.glfwMakeContextCurrent(main_window)
if g_pool.iconified:
pass
else:
g_pool.image_tex.update_from_frame(frame)
glFlush()
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
if not g_pool.iconified:
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil0_graph.draw()
pupil1_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture.close()
#shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'world_process.stopped'})
#shut down launcher
n = {'subject': 'launcher_process.should_stop'}
ipc_pub.notify(n)
zmq_ctx.destroy()
def thread_loop(self, context, pipe):
"""Background Relay Thread
1. Connects to the ZMQ backbone
2. Creates LSL outlets according to settings
3. Subscibes to according topics
4. Loop
4.1. Relay data
4.2. Handle un/subscription events
4.3. Handle exit event
5. Shutdown background thread
"""
try:
logger.debug('Connecting to %s...' % self.g_pool.ipc_sub_url)
inlet = zmq_tools.Msg_Receiver(context,
self.g_pool.ipc_sub_url,
block_until_connected=False)
poller = zmq.Poller()
poller.register(pipe, flags=zmq.POLLIN)
poller.register(inlet.socket, flags=zmq.POLLIN)
pupil_outlets = None
gaze_outlets = None
notification_outlet = None
# initial subscription
if self.relay_pupil:
pupil_outlets = self._create_pupil_lsl_outlets()
inlet.subscribe(PUPIL_SUB_TOPIC)
if self.relay_gaze:
gaze_outlets = self._create_gaze_lsl_outlets()
inlet.subscribe(GAZE_SUB_TOPIC)
if self.relay_notifications:
notification_outlet = self._create_notify_lsl_outlet()
inlet.subscribe(NOTIFY_SUB_TOPIC)
while True:
items = dict(poller.poll())
if inlet.socket in items:
topic, payload = inlet.recv()
if (topic.startswith(PUPIL_SUB_TOPIC) and pupil_outlets):
eyeid = payload['id']
# push primitive sample
outlet = pupil_outlets[eyeid * 2]
sample = self._generate_primitive_sample(payload)
outlet.push_sample(sample)
# push python sample
outlet = pupil_outlets[eyeid * 2 + 1]
outlet.push_sample((repr(payload), ))
del outlet # delete reference
elif (topic.startswith(GAZE_SUB_TOPIC) and gaze_outlets):
# push primitive sample
outlet = gaze_outlets[0]
sample = self._generate_primitive_sample(payload)
outlet.push_sample(sample)
# push python sample
outlet = gaze_outlets[1]
outlet.push_sample((repr(payload), ))
del outlet # delete reference
elif (topic.startswith(NOTIFY_SUB_TOPIC)
and notification_outlet):
sample = (payload['subject'], repr(payload))
notification_outlet.push_sample(sample)
else:
logger.debug('Did not handle topic "%s"' % topic)
if pipe in items:
cmd = pipe.recv_string()
if cmd == 'Exit':
break
elif cmd == 'Subscribe':
topic = pipe.recv_string()
inlet.subscribe(topic)
if topic == PUPIL_SUB_TOPIC and not pupil_outlets:
pupil_outlets = self._create_pupil_lsl_outlets()
elif topic == GAZE_SUB_TOPIC and not gaze_outlets:
gaze_outlets = self._create_gaze_lsl_outlets()
elif (topic == NOTIFY_SUB_TOPIC
and not notification_outlet):
notification_outlet = self._create_notify_lsl_outlet(
)
logger.debug('Subscribed to "%s"' % topic)
elif cmd == 'Unsubscribe':
topic = pipe.recv_string()
inlet.unsubscribe(topic)
if topic == PUPIL_SUB_TOPIC and pupil_outlets:
pupil_outlets = None
elif topic == GAZE_SUB_TOPIC and gaze_outlets:
gaze_outlets = None
elif topic == NOTIFY_SUB_TOPIC and notification_outlet:
notification_outlet = None
logger.debug('Unubscribed from "%s"' % topic)
except Exception as e:
logger.error('Error during relaying data to LSL. ' +
'Unloading the plugin...')
logger.debug('Error details: %s' % e)
finally:
logger.debug('Shutting down background thread...')
self.thread_pipe = None
self.alive = False
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version, debug):
# general imports
from time import sleep
import logging
from glob import glob
from time import time, strftime, localtime
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
from tasklib.background.patches import IPCLoggingPatch
IPCLoggingPatch.ipc_push_url = ipc_push_url
# imports
from file_methods import Persistent_Dict, next_export_sub_dir
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture, RGBA
import gl_utils
# capture
from video_capture import File_Source
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
import player_methods as pm
from pupil_recording import PupilRecording
from csv_utils import write_key_value_file
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from seek_control import Seek_Control
from surface_tracker import Surface_Tracker_Offline
# from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producer.gaze_from_recording import GazeFromRecording
from gaze_producer.gaze_from_offline_calibration import (
GazeFromOfflineCalibration, )
from system_graphs import System_Graphs
from system_timelines import System_Timelines
from blink_detection import Offline_Blink_Detection
from audio_playback import Audio_Playback
from video_export.plugins.imotions_exporter import iMotions_Exporter
from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
from video_export.plugins.world_video_exporter import World_Video_Exporter
from head_pose_tracker.offline_head_pose_tracker import (
Offline_Head_Pose_Tracker, )
from video_capture import File_Source
from video_overlay.plugins import Video_Overlay, Eye_Overlay
from pupil_recording import (
assert_valid_recording_type,
InvalidRecordingException,
)
assert VersionFormat(pyglui_version) >= VersionFormat(
"1.28"), "pyglui out of date, please upgrade to newest version"
process_was_interrupted = False
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
nonlocal process_was_interrupted
process_was_interrupted = True
signal.signal(signal.SIGINT, interrupt_handler)
runtime_plugins = import_runtime_plugins(
os.path.join(user_dir, "plugins"))
system_plugins = [
Log_Display,
Seek_Control,
Plugin_Manager,
System_Graphs,
System_Timelines,
Audio_Playback,
]
user_plugins = [
Vis_Circle,
Vis_Fixation,
Vis_Polyline,
Vis_Light_Points,
Vis_Cross,
Vis_Watermark,
Eye_Overlay,
Video_Overlay,
Offline_Fixation_Detector,
Offline_Blink_Detection,
Surface_Tracker_Offline,
Raw_Data_Exporter,
Annotation_Player,
Log_History,
Pupil_From_Recording,
Offline_Pupil_Detection,
GazeFromRecording,
GazeFromOfflineCalibration,
World_Video_Exporter,
iMotions_Exporter,
Eye_Video_Exporter,
Offline_Head_Pose_Tracker,
] + runtime_plugins
plugins = system_plugins + user_plugins
def consume_events_and_render_buffer():
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window,
user_input.clipboard.encode())
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = glfw.window_coordinate_to_framebuffer_coordinate(
main_window, x, y, cached_scale=None)
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
glfw.glfwSwapBuffers(main_window)
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal content_scale
if w == 0 or h == 0:
return
# Always clear buffers on resize to make sure that there are no overlapping
# artifacts from previous frames.
gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT)
gl_utils.glClearColor(0, 0, 0, 1)
content_scale = glfw.get_content_scale(window)
framebuffer_scale = glfw.get_framebuffer_scale(window)
g_pool.gui.scale = content_scale
window_size = w, h
g_pool.camera_render_size = w - int(
icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
# Minimum window size required, otherwise parts of the UI can cause openGL
# issues with permanent effects. Depends on the content scale, which can
# potentially be dynamically modified, so we re-adjust the size limits every
# time here.
min_size = int(2 * icon_bar_width * g_pool.gui.scale /
framebuffer_scale)
glfw.glfwSetWindowSizeLimits(window, min_size, min_size,
glfw.GLFW_DONT_CARE,
glfw.GLFW_DONT_CARE)
# Needed, to update the window buffer while resizing
consume_events_and_render_buffer()
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = glfw.window_coordinate_to_framebuffer_coordinate(
window, x, y, cached_scale=None)
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
for path in paths:
try:
assert_valid_recording_type(path)
_restart_with_recording(path)
return
except InvalidRecordingException as err:
logger.debug(str(err))
for plugin in g_pool.plugins:
if plugin.on_drop(paths):
break
def _restart_with_recording(rec_dir):
logger.debug("Starting new session with '{}'".format(rec_dir))
ipc_pub.notify({
"subject": "player_drop_process.should_start",
"rec_dir": rec_dir
})
glfw.glfwSetWindowShouldClose(g_pool.main_window, True)
tick = delta_t()
def get_dt():
return next(tick)
recording = PupilRecording(rec_dir)
meta_info = recording.meta_info
# log info about Pupil Platform and Platform in player.log
logger.info("Application Version: {}".format(app_version))
logger.info("System Info: {}".format(get_system_info()))
logger.debug(f"Debug flag: {debug}")
icon_bar_width = 50
window_size = None
content_scale = 1.0
# create container for globally scoped vars
g_pool = SimpleNamespace()
g_pool.app = "player"
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
video_path = recording.files().core().world().videos()[0].resolve()
File_Source(
g_pool,
timing="external",
source_path=video_path,
buffered_decoding=True,
fill_gaps=True,
)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version",
"0.0")) != app_version:
logger.info(
"Session setting are a different version of this app. I will not use those."
)
session_settings.clear()
width, height = g_pool.capture.frame_size
width += icon_bar_width
width, height = session_settings.get("window_size", (width, height))
window_pos = session_settings.get("window_position",
window_position_default)
window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}"
glfw.glfwInit()
glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE)
main_window = glfw.glfwCreateWindow(width, height, window_name, None,
None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.0
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get(
"min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT)
g_pool.min_calibration_confidence = session_settings.get(
"min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT)
# populated by producers
g_pool.pupil_positions = pm.PupilDataBisector()
g_pool.gaze_positions = pm.Bisector()
g_pool.fixations = pm.Affiliator()
g_pool.eye_movements = pm.Affiliator()
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {"subject": "min_data_confidence_changed"}
notification["_notify_time_"] = time() + 0.8
g_pool.ipc_pub.notify(notification)
def do_export(_):
left_idx = g_pool.seek_control.trim_left
right_idx = g_pool.seek_control.trim_right
export_range = left_idx, right_idx + 1 # exclusive range.stop
export_ts_window = pm.exact_window(g_pool.timestamps,
(left_idx, right_idx))
export_dir = os.path.join(g_pool.rec_dir, "exports")
export_dir = next_export_sub_dir(export_dir)
os.makedirs(export_dir)
logger.info('Created export dir at "{}"'.format(export_dir))
export_info = {
"Player Software Version":
str(g_pool.version),
"Data Format Version":
meta_info.min_player_version,
"Export Date":
strftime("%d.%m.%Y", localtime()),
"Export Time":
strftime("%H:%M:%S", localtime()),
"Frame Index Range:":
g_pool.seek_control.get_frame_index_trim_range_string(),
"Relative Time Range":
g_pool.seek_control.get_rel_time_trim_range_string(),
"Absolute Time Range":
g_pool.seek_control.get_abs_time_trim_range_string(),
}
with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
write_key_value_file(csv, export_info)
notification = {
"subject": "should_export",
"range": export_range,
"ts_window": export_ts_window,
"export_dir": export_dir,
}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({"subject": "clear_settings_process.should_start"})
ipc_pub.notify({
"subject": "player_process.should_start",
"rec_dir": rec_dir,
"delay": 2.0,
})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos="left")
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos="hidden")
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
g_pool.user_timelines = ui.Timeline_Menu("User Timelines",
pos=(0.0, -150.0),
size=(0.0, 0.0),
header_pos="headline")
g_pool.user_timelines.color = RGBA(a=0.0)
g_pool.user_timelines.collapsed = True
# add container that constaints itself to the seekbar height
vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
vert_constr.append(g_pool.user_timelines)
g_pool.timelines.append(vert_constr)
def set_window_size():
# Get current capture frame size
f_width, f_height = g_pool.capture.frame_size
# Get current display scale factor
content_scale = glfw.get_content_scale(main_window)
framebuffer_scale = glfw.get_framebuffer_scale(main_window)
display_scale_factor = content_scale / framebuffer_scale
# Scale the capture frame size by display scale factor
f_width *= display_scale_factor
f_height *= display_scale_factor
# Increas the width to account for the added scaled icon bar width
f_width += icon_bar_width * display_scale_factor
# Set the newly calculated size (scaled capture frame size + scaled icon bar width)
glfw.glfwSetWindowSize(main_window, int(f_width), int(f_height))
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(ui.Button("Reset window size",
set_window_size))
general_settings.append(
ui.Info_Text(
f"Minimum Player Version: {meta_info.min_player_version}"))
general_settings.append(
ui.Info_Text(f"Player Version: {g_pool.version}"))
general_settings.append(
ui.Info_Text(
f"Recording Software: {meta_info.recording_software_name}"))
general_settings.append(
ui.Info_Text(
f"Recording Software Version: {meta_info.recording_software_version}"
))
general_settings.append(
ui.Info_Text(
"High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
))
general_settings.append(
ui.Slider(
"min_data_confidence",
g_pool,
setter=set_data_confidence,
step=0.05,
min=0.0,
max=1.0,
label="Minimum data confidence",
))
general_settings.append(
ui.Button("Restart with default settings", reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
(100, -100))
g_pool.export_button = ui.Thumb(
"export",
label=chr(0xE2C5),
getter=lambda: False,
setter=do_export,
hotkey="e",
label_font="pupil_icons",
)
g_pool.quickbar.extend([g_pool.export_button])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [
("Plugin_Manager", {}),
("Seek_Control", {}),
("Log_Display", {}),
("Raw_Data_Exporter", {}),
("Vis_Polyline", {}),
("Vis_Circle", {}),
("System_Graphs", {}),
("System_Timelines", {}),
("World_Video_Exporter", {}),
("Pupil_From_Recording", {}),
("GazeFromRecording", {}),
("Audio_Playback", {}),
]
g_pool.plugins = Plugin_List(
g_pool, session_settings.get("loaded_plugins", default_plugins))
# Manually add g_pool.capture to the plugin list
g_pool.plugins._plugins.append(g_pool.capture)
g_pool.plugins._plugins.sort(key=lambda p: p.order)
g_pool.capture.init_ui()
general_settings.insert(
-1,
ui.Text_Input(
"rel_time_trim_section",
getter=g_pool.seek_control.get_rel_time_trim_range_string,
setter=g_pool.seek_control.set_rel_time_trim_range_string,
label="Relative time range to export",
),
)
general_settings.insert(
-1,
ui.Text_Input(
"frame_idx_trim_section",
getter=g_pool.seek_control.get_frame_index_trim_range_string,
setter=g_pool.seek_control.set_frame_index_trim_range_string,
label="Frame index range to export",
),
)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
toggle_general_settings(True)
g_pool.gui.configuration = session_settings.get("ui_config", {})
# If previously selected plugin was not loaded this time, we will have an
# expanded menubar without any menu selected. We need to ensure the menubar is
# collapsed in this case.
if all(submenu.collapsed for submenu in g_pool.menubar.elements):
g_pool.menubar.collapsed = True
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n["subject"]
if subject == "start_plugin":
g_pool.plugins.add(g_pool.plugin_by_name[n["name"]],
args=n.get("args", {}))
elif subject.startswith("meta.should_doc"):
ipc_pub.notify({
"subject": "meta.doc",
"actor": g_pool.app,
"doc": player.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
"subject": "meta.doc",
"actor": p.class_name,
"doc": p.on_notify.__doc__,
})
while (not glfw.glfwWindowShouldClose(main_window)
and not process_was_interrupted):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
events = {}
# report time between now and the last loop interation
events["dt"] = get_dt()
# pupil and gaze positions are added by their respective producer plugins
events["pupil"] = []
events["gaze"] = []
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
glfw.glfwPollEvents()
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(
main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window,
user_input.clipboard.encode())
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = glfw.window_coordinate_to_framebuffer_coordinate(
main_window, x, y, cached_scale=None)
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
# present frames at appropriate speed
g_pool.seek_control.wait(events["frame"].timestamp)
glfw.glfwSwapBuffers(main_window)
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["min_data_confidence"] = g_pool.min_data_confidence
session_settings[
"min_calibration_confidence"] = g_pool.min_calibration_confidence
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["window_position"] = glfw.glfwGetWindowPos(
main_window)
session_settings["version"] = str(g_pool.version)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
f_width, f_height = session_window_size
if platform.system() in ("Windows", "Linux"):
f_width, f_height = (
f_width / content_scale,
f_height / content_scale,
)
session_settings["window_size"] = int(f_width), int(f_height)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except Exception:
import traceback
trace = traceback.format_exc()
logger.error("Process Player crashed with trace:\n{}".format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({"subject": "player_process.stopped"})
sleep(1.0)
def launcher():
"""Starts eye processes. Hosts the IPC Backbone and Logging functions.
Reacts to notifications:
``launcher_process.should_stop``: Stops the launcher process
``eye_process.should_start``: Starts the eye process
"""
#Reliable msg dispatch to the IPC via push bridge.
def pull_pub(ipc_pub_url, pull):
ctx = zmq.Context.instance()
pub = ctx.socket(zmq.PUB)
pub.connect(ipc_pub_url)
while True:
m = pull.recv_multipart()
pub.send_multipart(m)
#The delay proxy handles delayed notififications.
def delay_proxy(ipc_pub_url, ipc_sub_url):
ctx = zmq.Context.instance()
sub = zmq_tools.Msg_Receiver(ctx, ipc_sub_url, ('delayed_notify', ))
pub = zmq_tools.Msg_Dispatcher(ctx, ipc_pub_url)
poller = zmq.Poller()
poller.register(sub.socket, zmq.POLLIN)
waiting_notifications = {}
while True:
if poller.poll(timeout=250):
#Recv new delayed notification and store it.
topic, n = sub.recv()
n['_notify_time_'] = time() + n['delay']
waiting_notifications[n['subject']] = n
#When a notifications time has come, pop from dict and send it as notification
for s, n in list(waiting_notifications.items()):
if n['_notify_time_'] < time():
del n['_notify_time_']
del n['delay']
del waiting_notifications[s]
pub.notify(n)
#Recv log records from other processes.
def log_loop(ipc_sub_url, log_level_debug):
import logging
#Get the root logger
logger = logging.getLogger()
#set log level
if log_level_debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
#Stream to file
fh = logging.FileHandler(os.path.join(user_dir, '{}.log'.format(app)),
mode='w')
fh.setFormatter(
logging.Formatter(
'%(asctime)s - %(processName)s - [%(levelname)s] %(name)s: %(message)s'
))
logger.addHandler(fh)
#Stream to console.
ch = logging.StreamHandler()
ch.setFormatter(
logging.Formatter(
'%(processName)s - [%(levelname)s] %(name)s: %(message)s'))
logger.addHandler(ch)
# IPC setup to receive log messages. Use zmq_tools.ZMQ_handler to send messages to here.
sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("logging", ))
while True:
topic, msg = sub.recv()
record = logging.makeLogRecord(msg)
logger.handle(record)
## IPC
timebase = Value(c_double, 0)
eyes_are_alive = Value(c_bool, 0), Value(c_bool, 0)
zmq_ctx = zmq.Context()
#Let the OS choose the IP and PORT
ipc_pub_url = 'tcp://*:*'
ipc_sub_url = 'tcp://*:*'
ipc_push_url = 'tcp://*:*'
# Binding IPC Backbone Sockets to URLs.
# They are used in the threads started below.
# Using them in the main thread is not allowed.
xsub_socket = zmq_ctx.socket(zmq.XSUB)
xsub_socket.bind(ipc_pub_url)
ipc_pub_url = xsub_socket.last_endpoint.decode('utf8').replace(
"0.0.0.0", "127.0.0.1")
xpub_socket = zmq_ctx.socket(zmq.XPUB)
xpub_socket.bind(ipc_sub_url)
ipc_sub_url = xpub_socket.last_endpoint.decode('utf8').replace(
"0.0.0.0", "127.0.0.1")
pull_socket = zmq_ctx.socket(zmq.PULL)
pull_socket.bind(ipc_push_url)
ipc_push_url = pull_socket.last_endpoint.decode('utf8').replace(
"0.0.0.0", "127.0.0.1")
# Starting communication threads:
# A ZMQ Proxy Device serves as our IPC Backbone
ipc_backbone_thread = Thread(target=zmq.proxy,
args=(xsub_socket, xpub_socket))
ipc_backbone_thread.setDaemon(True)
ipc_backbone_thread.start()
pull_pub = Thread(target=pull_pub, args=(ipc_pub_url, pull_socket))
pull_pub.setDaemon(True)
pull_pub.start()
log_thread = Thread(target=log_loop,
args=(ipc_sub_url, 'debug' in sys.argv))
log_thread.setDaemon(True)
log_thread.start()
delay_thread = Thread(target=delay_proxy, args=(ipc_push_url, ipc_sub_url))
delay_thread.setDaemon(True)
delay_thread.start()
del xsub_socket, xpub_socket, pull_socket
topics = ('notify.eye_process.', 'notify.player_process.',
'notify.world_process.', 'notify.service_process',
'notify.reset_restart_process.', 'notify.player_drop_process.',
'notify.launcher_process.', 'notify.meta.should_doc',
'notify.circle_detector_process.should_start',
'notify.ipc_startup')
cmd_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=topics)
cmd_push = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
while True:
# Wait until subscriptions were successfull
cmd_push.notify({'subject': 'ipc_startup'})
if cmd_sub.socket.poll(timeout=50):
cmd_sub.recv()
break
if app == 'service':
cmd_push.notify({'subject': 'service_process.should_start'})
elif app == 'capture':
cmd_push.notify({'subject': 'world_process.should_start'})
elif app == 'player':
if len(sys.argv) > 2:
rec_dir = os.path.expanduser(sys.argv[-1])
else:
rec_dir = None
cmd_push.notify({
'subject': 'player_drop_process.should_start',
'rec_dir': rec_dir
})
with Prevent_Idle_Sleep():
while True:
# listen for relevant messages.
if cmd_sub.socket.poll(timeout=1000):
topic, n = cmd_sub.recv()
if "notify.eye_process.should_start" in topic:
eye_id = n['eye_id']
Process(target=eye,
name='eye{}'.format(eye_id),
args=(timebase, eyes_are_alive[eye_id],
ipc_pub_url, ipc_sub_url, ipc_push_url,
user_dir, app_version, eye_id,
n.get('overwrite_cap_settings'))).start()
elif "notify.player_process.should_start" in topic:
Process(target=player,
name='player',
args=(
n['rec_dir'],
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
)).start()
elif "notify.world_process.should_start" in topic:
Process(target=world,
name='world',
args=(
timebase,
eyes_are_alive,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
)).start()
elif "notify.reset_restart_process.should_start" in topic:
Process(target=reset_restart,
name='reset_restart',
args=(
ipc_push_url,
user_dir,
)).start()
elif "notify.service_process.should_start" in topic:
Process(target=service,
name='service',
args=(timebase, eyes_are_alive, ipc_pub_url,
ipc_sub_url, ipc_push_url, user_dir,
app_version)).start()
elif "notify.player_drop_process.should_start" in topic:
Process(target=player_drop,
name='player',
args=(
n['rec_dir'],
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
)).start()
elif "notify.circle_detector_process.should_start" in topic:
Process(target=circle_detector,
name='circle_detector',
args=(ipc_push_url, n['pair_url'],
n['source_path'])).start()
elif "notify.meta.should_doc" in topic:
cmd_push.notify({
'subject': 'meta.doc',
'actor': 'launcher',
'doc': launcher.__doc__
})
else:
if not active_children():
break
for p in active_children():
p.join()
def thread_loop(self, context, pipe):
poller = zmq.Poller()
ipc_pub = zmq_tools.Msg_Dispatcher(context, self.g_pool.ipc_push_url)
ipc_sub = zmq_tools.Msg_Receiver(context, self.g_pool.ipc_sub_url,
('gaze', 'notify.calibration.failed',
'notify.calibration.successful'))
poller.register(pipe, zmq.POLLIN)
poller.register(ipc_sub.socket, zmq.POLLIN)
remote_socket = None
self.gaze_receiver = None
self.calib_result_receiver = None
while True:
items = [sock for sock, _ in poller.poll()]
if pipe in items:
cmd = pipe.recv_string()
if cmd == 'Exit':
break
elif cmd == 'Bind':
new_url = pipe.recv_string()
if remote_socket:
poller.unregister(remote_socket)
remote_socket.close()
try:
remote_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
remote_socket.setblocking(False)
addr, port = new_url.split(':')
socket_addr = (addr, int(port))
remote_socket.bind(socket_addr)
except OSError as e:
remote_socket = None
pipe.send_string("Error", flags=zmq.SNDMORE)
pipe.send_string("Could not bind to Socket: {}. Reason: {}".format(new_url, e))
else:
pipe.send_string("Bind OK", flags=zmq.SNDMORE)
# TODO: get addr
pipe.send_string(new_url)
poller.register(remote_socket, zmq.POLLIN)
if remote_socket.fileno() in items:
self.on_recv(remote_socket, ipc_pub)
if ipc_sub.socket in items:
while ipc_sub.new_data:
# we should only receive gaze data
topic, payload = ipc_sub.recv()
# gaze events
if (self.gaze_receiver is not None and
remote_socket is not None and
topic.startswith('gaze')):
_, method, eye = payload['topic'].split('.')
if method == '2d':
data = b'EG%s%s%s' % (method[:1].encode(),
eye.encode(),
struct.pack('ff', *payload['norm_pos']))
elif method == '3d':
data = b'EG%s%s%s' % (method[:1].encode(),
eye.encode(),
struct.pack('fff', *payload['gaze_point_3d']))
else:
logger.error('Error while relaying gaze: "{}": {}'.format(topic, payload))
data = b'EGFError while relaying gaze'
remote_socket.sendto(data, self.gaze_receiver)
# calibration events
elif (self.calib_result_receiver is not None and
remote_socket is not None and
topic.startswith('notify.calibration.')):
if payload['subject'] == 'calibration.successful':
# event calibration successful
remote_socket.sendto(b'ECS', self.calib_result_receiver)
self.calib_result_receiver = None
elif payload['subject'] == 'calibration.failed':
# event calibration failed
remote_socket.sendto(b'ECF', self.calib_result_receiver)
self.calib_result_receiver = None
remote_socket.close()
self.thread_pipe = None
def world(
timebase,
eye_procs_alive,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
preferred_remote_port,
hide_ui,
debug,
):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``eye_process.started``
``start_plugin``
``should_stop``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events
that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
# general imports
from time import sleep
import logging
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
def launch_eye_process(eye_id, delay=0):
n = {
"subject": "eye_process.should_start.{}".format(eye_id),
"eye_id": eye_id,
"delay": delay,
}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {
"subject": "eye_process.should_stop.{}".format(eye_id),
"eye_id": eye_id,
"delay": 0.2,
}
ipc_pub.notify(n)
def start_stop_eye(eye_id, make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def detection_enabled_getter() -> bool:
return g_pool.pupil_detection_enabled
def detection_enabled_setter(is_on: bool):
g_pool.pupil_detection_enabled = is_on
n = {"subject": "set_pupil_detection_enabled", "value": is_on}
ipc_pub.notify(n)
try:
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
from tasklib.background.patches import IPCLoggingPatch
IPCLoggingPatch.ipc_push_url = ipc_push_url
from OpenGL.GL import GL_COLOR_BUFFER_BIT
# display
import glfw
glfw.ERROR_REPORTING = "raise"
from version_utils import parse_version
from pyglui import ui, cygl, __version__ as pyglui_version
assert parse_version(pyglui_version) >= parse_version(
"1.27"
), "pyglui out of date, please upgrade to newest version"
from pyglui.cygl.utils import Named_Texture
import gl_utils
# helpers/utils
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info, timer
from uvc import get_time_monotonic
logger.info("Application Version: {}".format(version))
logger.info("System Info: {}".format(get_system_info()))
logger.debug(f"Debug flag: {debug}")
import audio
# Plug-ins
from plugin import (
Plugin,
System_Plugin_Base,
Plugin_List,
import_runtime_plugins,
)
from plugin_manager import Plugin_Manager
from calibration_choreography import (
available_calibration_choreography_plugins,
CalibrationChoreographyPlugin,
patch_loaded_plugins_with_choreography_plugin,
)
available_choreography_plugins = available_calibration_choreography_plugins()
from gaze_mapping import registered_gazer_classes
from gaze_mapping.gazer_base import GazerBase
from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin
from fixation_detector import Fixation_Detector
from recorder import Recorder
from display_recent_gaze import Display_Recent_Gaze
from time_sync import Time_Sync
from network_api import NetworkApiPlugin
from pupil_groups import Pupil_Groups
from surface_tracker import Surface_Tracker_Online
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
from blink_detection import Blink_Detection
from video_capture import (
source_classes,
manager_classes,
Base_Manager,
Base_Source,
)
from pupil_data_relay import Pupil_Data_Relay
from remote_recorder import Remote_Recorder
from accuracy_visualizer import Accuracy_Visualizer
from system_graphs import System_Graphs
from camera_intrinsics_estimation import Camera_Intrinsics_Estimation
from hololens_relay import Hololens_Relay
from head_pose_tracker.online_head_pose_tracker import Online_Head_Pose_Tracker
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (30, 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (8, 90)
else:
scroll_factor = 1.0
window_position_default = (0, 0)
process_was_interrupted = False
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
nonlocal process_was_interrupted
process_was_interrupted = True
signal.signal(signal.SIGINT, interrupt_handler)
icon_bar_width = 50
window_size = None
camera_render_size = None
content_scale = 1.0
# g_pool holds variables for this process they are accessible to all plugins
g_pool = SimpleNamespace()
g_pool.debug = debug
g_pool.app = "capture"
g_pool.process = "world"
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eye_procs_alive = eye_procs_alive
g_pool.preferred_remote_port = preferred_remote_port
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, "plugins")
)
runtime_plugins = [
p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin)
]
user_plugins = [
Pupil_Groups,
NetworkApiPlugin,
Time_Sync,
Surface_Tracker_Online,
Annotation_Capture,
Log_History,
Fixation_Detector,
Blink_Detection,
Remote_Recorder,
Accuracy_Visualizer,
Camera_Intrinsics_Estimation,
Hololens_Relay,
Online_Head_Pose_Tracker,
]
system_plugins = (
[
Log_Display,
Display_Recent_Gaze,
Recorder,
Pupil_Data_Relay,
Plugin_Manager,
System_Graphs,
]
+ manager_classes
+ source_classes
)
plugins = (
system_plugins
+ user_plugins
+ runtime_plugins
+ available_choreography_plugins
+ registered_gazer_classes()
)
user_plugins += [
p
for p in runtime_plugins
if not isinstance(
p,
(
Base_Manager,
Base_Source,
System_Plugin_Base,
CalibrationChoreographyPlugin,
GazerBase,
),
)
]
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
default_capture_name = "UVC_Source"
default_capture_settings = {
"preferred_names": [
"Pupil Cam1 ID2",
"Logitech Camera",
"(046d:081d)",
"C510",
"B525",
"C525",
"C615",
"C920",
"C930e",
],
"frame_size": (1280, 720),
"frame_rate": 30,
}
default_plugins = [
(default_capture_name, default_capture_settings),
("Pupil_Data_Relay", {}),
("UVC_Manager", {}),
("NDSI_Manager", {}),
("HMD_Streaming_Manager", {}),
("File_Manager", {}),
("Log_Display", {}),
("Dummy_Gaze_Mapper", {}),
("Display_Recent_Gaze", {}),
# Calibration choreography plugin is added below by calling
# patch_loaded_plugins_with_choreography_plugin
("Recorder", {}),
("NetworkApiPlugin", {}),
("Fixation_Detector", {}),
("Blink_Detection", {}),
("Accuracy_Visualizer", {}),
("Plugin_Manager", {}),
("System_Graphs", {}),
]
def consume_events_and_render_buffer():
gl_utils.glViewport(0, 0, *camera_render_size)
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.get_clipboard_string(main_window).decode()
except (AttributeError, glfw.GLFWError):
# clipboard is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.set_clipboard_string(main_window, user_input.clipboard)
for button, action, mods in user_input.buttons:
x, y = glfw.get_cursor_pos(main_window)
pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
main_window, x, y, cached_scale=None
)
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
glfw.swap_buffers(main_window)
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal content_scale
if w == 0 or h == 0:
return
# Always clear buffers on resize to make sure that there are no overlapping
# artifacts from previous frames.
gl_utils.glClear(GL_COLOR_BUFFER_BIT)
gl_utils.glClearColor(0, 0, 0, 1)
content_scale = gl_utils.get_content_scale(window)
framebuffer_scale = gl_utils.get_framebuffer_scale(window)
g_pool.gui.scale = content_scale
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *camera_render_size)
# Minimum window size required, otherwise parts of the UI can cause openGL
# issues with permanent effects. Depends on the content scale, which can
# potentially be dynamically modified, so we re-adjust the size limits every
# time here.
min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale)
glfw.set_window_size_limits(
window,
min_size,
min_size,
glfw.DONT_CARE,
glfw.DONT_CARE,
)
# Needed, to update the window buffer while resizing
consume_events_and_render_buffer()
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = gl_utils.window_coordinate_to_framebuffer_coordinate(
window, x, y, cached_scale=None
)
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, paths):
for plugin in g_pool.plugins:
if plugin.on_drop(paths):
break
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, "user_settings_world")
)
if parse_version(session_settings.get("version", "0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.min_data_confidence = 0.6
g_pool.min_calibration_confidence = session_settings.get(
"min_calibration_confidence", 0.8
)
g_pool.pupil_detection_enabled = session_settings.get(
"pupil_detection_enabled", True
)
g_pool.active_gaze_mapping_plugin = None
g_pool.capture = None
audio.set_audio_mode(
session_settings.get("audio_mode", audio.get_default_audio_mode())
)
def handle_notifications(noti):
subject = noti["subject"]
if subject == "set_pupil_detection_enabled":
g_pool.pupil_detection_enabled = noti["value"]
elif subject == "start_plugin":
try:
g_pool.plugins.add(
g_pool.plugin_by_name[noti["name"]], args=noti.get("args", {})
)
except KeyError as err:
logger.error(f"Attempt to load unknown plugin: {err}")
elif subject == "stop_plugin":
for p in g_pool.plugins:
if p.class_name == noti["name"]:
p.alive = False
g_pool.plugins.clean()
elif subject == "eye_process.started":
noti = {
"subject": "set_pupil_detection_enabled",
"value": g_pool.pupil_detection_enabled,
}
ipc_pub.notify(noti)
elif subject == "set_min_calibration_confidence":
g_pool.min_calibration_confidence = noti["value"]
elif subject.startswith("meta.should_doc"):
ipc_pub.notify(
{"subject": "meta.doc", "actor": g_pool.app, "doc": world.__doc__}
)
for p in g_pool.plugins:
if (
p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify
):
ipc_pub.notify(
{
"subject": "meta.doc",
"actor": p.class_name,
"doc": p.on_notify.__doc__,
}
)
elif subject == "world_process.adapt_window_size":
set_window_size()
elif subject == "world_process.should_stop":
glfw.set_window_should_close(main_window, True)
width, height = session_settings.get(
"window_size", (1280 + icon_bar_width, 720)
)
# window and gl setup
glfw.init()
glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE)
if hide_ui:
glfw.window_hint(glfw.VISIBLE, 0) # hide window
main_window = glfw.create_window(
width, height, "Pupil Capture - World", None, None
)
window_position_manager = gl_utils.WindowPositionManager()
window_pos = window_position_manager.new_window_position(
window=main_window,
default_position=window_position_default,
previous_position=session_settings.get("window_position", None),
)
glfw.set_window_pos(main_window, window_pos[0], window_pos[1])
glfw.make_context_current(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def reset_restart():
logger.warning("Resetting all settings and restarting Capture.")
glfw.set_window_should_close(main_window, True)
ipc_pub.notify({"subject": "clear_settings_process.should_start"})
ipc_pub.notify({"subject": "world_process.should_start", "delay": 2.0})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is opened, the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# setup GUI
g_pool.gui = ui.UI()
g_pool.menubar = ui.Scrolling_Menu(
"Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left"
)
g_pool.iconbar = ui.Scrolling_Menu(
"Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
)
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (120, -100))
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
def set_window_size():
# Get current capture frame size
f_width, f_height = g_pool.capture.frame_size
# Get current display scale factor
content_scale = gl_utils.get_content_scale(main_window)
framebuffer_scale = gl_utils.get_framebuffer_scale(main_window)
display_scale_factor = content_scale / framebuffer_scale
# Scale the capture frame size by display scale factor
f_width *= display_scale_factor
f_height *= display_scale_factor
# Increas the width to account for the added scaled icon bar width
f_width += icon_bar_width * display_scale_factor
# Set the newly calculated size (scaled capture frame size + scaled icon bar width)
glfw.set_window_size(main_window, int(f_width), int(f_height))
general_settings.append(ui.Button("Reset window size", set_window_size))
general_settings.append(
ui.Selector(
"Audio mode",
None,
getter=audio.get_audio_mode,
setter=audio.set_audio_mode,
selection=audio.get_audio_mode_list(),
)
)
general_settings.append(
ui.Switch(
"pupil_detection_enabled",
label="Pupil detection",
getter=detection_enabled_getter,
setter=detection_enabled_setter,
)
)
general_settings.append(
ui.Switch(
"eye0_process",
label="Detect eye 0",
setter=lambda alive: start_stop_eye(0, alive),
getter=lambda: eye_procs_alive[0].value,
)
)
general_settings.append(
ui.Switch(
"eye1_process",
label="Detect eye 1",
setter=lambda alive: start_stop_eye(1, alive),
getter=lambda: eye_procs_alive[1].value,
)
)
general_settings.append(
ui.Info_Text("Capture Version: {}".format(g_pool.version))
)
general_settings.append(
ui.Button("Restart with default settings", reset_restart)
)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
loaded_plugins = session_settings.get("loaded_plugins", default_plugins)
# Resolve the active calibration choreography plugin
loaded_plugins = patch_loaded_plugins_with_choreography_plugin(
loaded_plugins, app=g_pool.app
)
session_settings["loaded_plugins"] = loaded_plugins
# plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(g_pool, loaded_plugins)
if not g_pool.capture:
# Make sure we always have a capture running. Important if there was no
# capture stored in session settings.
g_pool.plugins.add(
g_pool.plugin_by_name[default_capture_name], default_capture_settings
)
# Register callbacks main_window
glfw.set_framebuffer_size_callback(main_window, on_resize)
glfw.set_key_callback(main_window, on_window_key)
glfw.set_char_callback(main_window, on_window_char)
glfw.set_mouse_button_callback(main_window, on_window_mouse_button)
glfw.set_cursor_pos_callback(main_window, on_pos)
glfw.set_scroll_callback(main_window, on_scroll)
glfw.set_drop_callback(main_window, on_drop)
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
toggle_general_settings(True)
# now that we have a proper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# If previously selected plugin was not loaded this time, we will have an
# expanded menubar without any menu selected. We need to ensure the menubar is
# collapsed in this case.
if all(submenu.collapsed for submenu in g_pool.menubar.elements):
g_pool.menubar.collapsed = True
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
# trigger setup of window and gl sizes
on_resize(main_window, *glfw.get_framebuffer_size(main_window))
if session_settings.get("eye1_process_alive", True):
launch_eye_process(1, delay=0.6)
if session_settings.get("eye0_process_alive", True):
launch_eye_process(0, delay=0.3)
ipc_pub.notify({"subject": "world_process.started"})
logger.warning("Process started.")
if platform.system() == "Darwin":
# On macOS, calls to glfw.swap_buffers() deliberately take longer in case of
# occluded windows, based on the swap interval value. This causes an FPS drop
# and leads to problems when recording. To side-step this behaviour, the swap
# interval is set to zero.
#
# Read more about window occlusion on macOS here:
# https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/WorkWhenVisible.html
glfw.swap_interval(0)
# Event loop
while not glfw.window_should_close(main_window) and not process_was_interrupted:
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# a dictionary that allows plugins to post and read events
events = {}
# report time between now and the last loop interation
events["dt"] = get_dt()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# "blacklisted" events that were already sent
del events["pupil"]
del events["gaze"]
# delete if exists. More expensive than del, so only use it when key might not exist
events.pop("annotation", None)
# send new events to ipc:
if "frame" in events:
del events["frame"] # send explicitly with frame publisher
if "depth_frame" in events:
del events["depth_frame"]
if "audio_packets" in events:
del events["audio_packets"]
del events["dt"] # no need to send this
for data in events.values():
assert isinstance(data, (list, tuple))
for d in data:
ipc_pub.send(d)
glfw.make_context_current(main_window)
# render visual feedback from loaded plugins
glfw.poll_events()
if window_should_update() and gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *camera_render_size)
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.get_clipboard_string(main_window).decode()
except (AttributeError, glfw.GLFWError):
# clipboard is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.set_clipboard_string(main_window, user_input.clipboard)
for button, action, mods in user_input.buttons:
x, y = glfw.get_cursor_pos(main_window)
pos = gl_utils.window_coordinate_to_framebuffer_coordinate(
main_window, x, y, cached_scale=None
)
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
glfw.swap_buffers(main_window)
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["version"] = str(g_pool.version)
session_settings["eye0_process_alive"] = eye_procs_alive[0].value
session_settings["eye1_process_alive"] = eye_procs_alive[1].value
session_settings[
"min_calibration_confidence"
] = g_pool.min_calibration_confidence
session_settings["pupil_detection_enabled"] = g_pool.pupil_detection_enabled
session_settings["audio_mode"] = audio.get_audio_mode()
if not hide_ui:
glfw.restore_window(main_window) # need to do this for windows os
session_settings["window_position"] = glfw.get_window_pos(main_window)
session_window_size = glfw.get_window_size(main_window)
if 0 not in session_window_size:
f_width, f_height = session_window_size
if platform.system() in ("Windows", "Linux"):
f_width, f_height = (
f_width / content_scale,
f_height / content_scale,
)
session_settings["window_size"] = int(f_width), int(f_height)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.destroy_window(main_window)
glfw.terminate()
except Exception:
import traceback
trace = traceback.format_exc()
logger.error("Process Capture crashed with trace:\n{}".format(trace))
finally:
# shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({"subject": "world_process.stopped"})
sleep(1.0)
def launcher():
"""Starts eye processes. Hosts the IPC Backbone and Logging functions.
Reacts to notifications:
``launcher_process.should_stop``: Stops the launcher process
``eye_process.should_start``: Starts the eye process
"""
# Reliable msg dispatch to the IPC via push bridge.
def pull_pub(ipc_pub_url, pull):
ctx = zmq.Context.instance()
pub = ctx.socket(zmq.PUB)
pub.connect(ipc_pub_url)
while True:
m = pull.recv_multipart()
pub.send_multipart(m)
# The delay proxy handles delayed notififications.
def delay_proxy(ipc_pub_url, ipc_sub_url):
ctx = zmq.Context.instance()
sub = zmq_tools.Msg_Receiver(ctx, ipc_sub_url, ("delayed_notify", ))
pub = zmq_tools.Msg_Dispatcher(ctx, ipc_pub_url)
poller = zmq.Poller()
poller.register(sub.socket, zmq.POLLIN)
waiting_notifications = {}
TOPIC_CUTOFF = len("delayed_")
while True:
if poller.poll(timeout=250):
# Recv new delayed notification and store it.
topic, n = sub.recv()
n["__notify_time__"] = time() + n["delay"]
waiting_notifications[n["subject"]] = n
# When a notifications time has come, pop from dict and send it as notification
for s, n in list(waiting_notifications.items()):
if n["__notify_time__"] < time():
n["topic"] = n["topic"][TOPIC_CUTOFF:]
del n["__notify_time__"]
del n["delay"]
del waiting_notifications[s]
pub.notify(n)
# Recv log records from other processes.
def log_loop(ipc_sub_url, log_level_debug):
import logging
# Get the root logger
logger = logging.getLogger()
# set log level
logger.setLevel(logging.NOTSET)
# Stream to file
fh = logging.FileHandler(
os.path.join(user_dir, "{}.log".format(parsed_args.app)),
mode="w",
encoding="utf-8",
)
fh.setFormatter(
logging.Formatter(
"%(asctime)s - %(processName)s - [%(levelname)s] %(name)s: %(message)s"
))
logger.addHandler(fh)
# Stream to console.
ch = logging.StreamHandler()
ch.setFormatter(
logging.Formatter(
"%(processName)s - [%(levelname)s] %(name)s: %(message)s"))
if log_level_debug:
ch.setLevel(logging.DEBUG)
else:
ch.setLevel(logging.INFO)
logger.addHandler(ch)
# IPC setup to receive log messages. Use zmq_tools.ZMQ_handler to send messages to here.
sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("logging", ))
while True:
topic, msg = sub.recv()
record = logging.makeLogRecord(msg)
logger.handle(record)
## IPC
timebase = Value(c_double, 0)
eye_procs_alive = Value(c_bool, 0), Value(c_bool, 0)
zmq_ctx = zmq.Context()
# Let the OS choose the IP and PORT
ipc_pub_url = "tcp://*:*"
ipc_sub_url = "tcp://*:*"
ipc_push_url = "tcp://*:*"
# Binding IPC Backbone Sockets to URLs.
# They are used in the threads started below.
# Using them in the main thread is not allowed.
xsub_socket = zmq_ctx.socket(zmq.XSUB)
xsub_socket.bind(ipc_pub_url)
ipc_pub_url = xsub_socket.last_endpoint.decode("utf8").replace(
"0.0.0.0", "127.0.0.1")
xpub_socket = zmq_ctx.socket(zmq.XPUB)
xpub_socket.bind(ipc_sub_url)
ipc_sub_url = xpub_socket.last_endpoint.decode("utf8").replace(
"0.0.0.0", "127.0.0.1")
pull_socket = zmq_ctx.socket(zmq.PULL)
pull_socket.bind(ipc_push_url)
ipc_push_url = pull_socket.last_endpoint.decode("utf8").replace(
"0.0.0.0", "127.0.0.1")
# Starting communication threads:
# A ZMQ Proxy Device serves as our IPC Backbone
ipc_backbone_thread = Thread(target=zmq.proxy,
args=(xsub_socket, xpub_socket))
ipc_backbone_thread.setDaemon(True)
ipc_backbone_thread.start()
pull_pub = Thread(target=pull_pub, args=(ipc_pub_url, pull_socket))
pull_pub.setDaemon(True)
pull_pub.start()
log_thread = Thread(target=log_loop, args=(ipc_sub_url, parsed_args.debug))
log_thread.setDaemon(True)
log_thread.start()
delay_thread = Thread(target=delay_proxy, args=(ipc_push_url, ipc_sub_url))
delay_thread.setDaemon(True)
delay_thread.start()
del xsub_socket, xpub_socket, pull_socket
topics = (
"notify.eye_process.",
"notify.player_process.",
"notify.world_process.",
"notify.service_process",
"notify.clear_settings_process.",
"notify.player_drop_process.",
"notify.launcher_process.",
"notify.meta.should_doc",
"notify.circle_detector_process.should_start",
"notify.ipc_startup",
)
cmd_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=topics)
cmd_push = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
while True:
# Wait until subscriptions were successfull
cmd_push.notify({"subject": "ipc_startup"})
if cmd_sub.socket.poll(timeout=50):
cmd_sub.recv()
break
import logging
if unknown_args:
logging.warning(f"Unknown command-line arguments: {unknown_args}")
if parsed_args.app == "service":
cmd_push.notify({"subject": "service_process.should_start"})
elif parsed_args.app == "capture":
cmd_push.notify({"subject": "world_process.should_start"})
elif parsed_args.app == "player":
rec_dir = os.path.expanduser(parsed_args.recording)
cmd_push.notify({
"subject": "player_drop_process.should_start",
"rec_dir": rec_dir
})
with Prevent_Idle_Sleep():
while True:
# listen for relevant messages.
if cmd_sub.socket.poll(timeout=1000):
topic, n = cmd_sub.recv()
if "notify.eye_process.should_start" in topic:
eye_id = n["eye_id"]
Process(
target=eye,
name="eye{}".format(eye_id),
args=(
timebase,
eye_procs_alive[eye_id],
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
eye_id,
n.get("overwrite_cap_settings"),
parsed_args.hide_ui,
),
).start()
elif "notify.player_process.should_start" in topic:
Process(
target=player,
name="player",
args=(
n["rec_dir"],
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
),
).start()
elif "notify.world_process.should_start" in topic:
Process(
target=world,
name="world",
args=(
timebase,
eye_procs_alive,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
parsed_args.port,
parsed_args.hide_ui,
),
).start()
elif "notify.clear_settings_process.should_start" in topic:
Process(target=clear_settings,
name="clear_settings",
args=(user_dir, )).start()
elif "notify.service_process.should_start" in topic:
Process(
target=service,
name="service",
args=(
timebase,
eye_procs_alive,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
parsed_args.port,
parsed_args.hide_ui,
),
).start()
elif "notify.player_drop_process.should_start" in topic:
Process(
target=player_drop,
name="player",
args=(
n["rec_dir"],
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
),
).start()
elif "notify.circle_detector_process.should_start" in topic:
Process(
target=circle_detector,
name="circle_detector",
args=(ipc_push_url, n["pair_url"], n["source_path"]),
).start()
elif "notify.meta.should_doc" in topic:
cmd_push.notify({
"subject": "meta.doc",
"actor": "launcher",
"doc": launcher.__doc__,
})
else:
if not active_children():
break
for p in active_children():
p.join()
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version):
# general imports
from time import sleep
import logging
import errno
from glob import glob
from copy import deepcopy
from time import time
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('notify', ))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
# imports
from file_methods import Persistent_Dict, load_object
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture
import gl_utils
# capture
from video_capture import File_Source, EndofVideoFileError, FileSeekError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_control import Seek_Control
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from batch_exporter import Batch_Exporter
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
from system_graphs import System_Graphs
assert pyglui_version >= '1.8', 'pyglui out of date, please upgrade to newest version'
runtime_plugins = import_runtime_plugins(
os.path.join(user_dir, 'plugins'))
system_plugins = [
Log_Display, Seek_Control, Plugin_Manager, System_Graphs
]
user_plugins = [
Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points,
Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path,
Offline_Fixation_Detector, Video_Export_Launcher,
Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter,
Annotation_Player, Log_History, Marker_Auto_Trim_Marks,
Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording,
Offline_Calibration
] + runtime_plugins
plugins = system_plugins + user_plugins
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
if gl_utils.is_window_visible(window):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
g_pool.camera_render_size = w - int(
icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug(
"Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({
"subject": "player_drop_process.should_start",
"rec_dir": new_rec_dir
})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(
new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264',
'.mjpeg')
][0]
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
icon_bar_width = 50
window_size = None
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
# sets itself to g_pool.capture
File_Source(g_pool, video_path)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version",
'0.0')) != app_version:
logger.info(
"Session setting are a different version of this app. I will not use those."
)
session_settings.clear()
width, height = session_settings.get('window_size',
g_pool.capture.frame_size)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] +
" - " + rec_dir.split(os.path.sep)[-1], None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
hdpi_factor = float(glfw.glfwGetFramebufferSize(main_window)
[0]) / glfw.glfwGetWindowSize(main_window)[0]
g_pool.gui_user_scale = new_scale
window_size = (
g_pool.camera_render_size[0] +
int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1])
logger.warning(icon_bar_width * g_pool.gui_user_scale *
hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode',
'monocular') == 'binocular'
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get(
'min_data_confidence', 0.6)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(
g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps
] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps
] # populated by producer
g_pool.fixations_by_frame = [
[] for x in g_pool.timestamps
] # populated by the fixation detector plugin
def next_frame(_):
try:
g_pool.capture.seek_to_frame(g_pool.capture.get_frame_index() +
1)
except (FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
g_pool.capture.seek_to_frame(g_pool.capture.get_frame_index() -
1)
except (FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if g_pool.capture.get_frame_index(
) >= g_pool.capture.get_frame_count() - 5:
g_pool.capture.seek_to_frame(
1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.capture.play = new_state
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time() + .8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right
export_dir = os.path.join(g_pool.rec_dir, 'exports',
'{}-{}'.format(*export_range))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(*export_range))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {
'subject': 'should_export',
'range': export_range,
'export_dir': export_dir
}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
ipc_pub.notify({
'subject': 'player_process.should_start',
'rec_dir': rec_dir,
'delay': 2.
})
def toggle_general_settings(collapsed):
#this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos='left')
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos='hidden')
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
general_settings = ui.Growing_Menu('General', header_pos='headline')
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, g_pool.capture.frame_size[0], g_pool.capture.
frame_size[1])))
general_settings.append(
ui.Selector('gui_user_scale',
g_pool,
setter=set_scale,
selection=[.8, .9, 1., 1.1, 1.2] +
list(np.arange(1.5, 5.1, .5)),
label='Interface Size'))
general_settings.append(
ui.Info_Text('Player Version: {}'.format(g_pool.version)))
general_settings.append(
ui.Info_Text('Capture Version: {}'.format(
meta_info['Capture Software Version'])))
general_settings.append(
ui.Info_Text('Data Format Version: {}'.format(
meta_info['Data Format Version'])))
general_settings.append(
ui.Slider('min_data_confidence',
g_pool,
setter=set_data_confidence,
step=.05,
min=0.0,
max=1.0,
label='Confidence threshold'))
general_settings.append(
ui.Button('Restart with default settings', reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon('collapsed',
general_settings,
label=chr(0xe8b8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font='pupil_icons')
icon.tooltip = 'General Settings'
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
(120, -100))
g_pool.capture.play_button = ui.Thumb('play',
g_pool.capture,
label=chr(0xE037),
setter=toggle_play,
hotkey=glfw.GLFW_KEY_SPACE,
label_font='pupil_icons')
g_pool.capture.play_button.on_color[:] = (0.5, 0.8, 0.75, .9)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xE01F),
getter=lambda: False,
setter=next_frame,
hotkey=glfw.GLFW_KEY_RIGHT,
label_font='pupil_icons')
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xE020),
getter=lambda: False,
setter=prev_frame,
hotkey=glfw.GLFW_KEY_LEFT,
label_font='pupil_icons')
g_pool.export_button = ui.Thumb('export',
label=chr(0xe2c4),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='pupil_icons')
g_pool.quickbar.extend([
g_pool.capture.play_button, g_pool.forward_button,
g_pool.backward_button, g_pool.export_button
])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}),
('Log_Display', {}), ('Vis_Scan_Path', {}),
('Vis_Polyline', {}), ('Vis_Circle', {}),
('System_Graphs', {}),
('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}),
('Gaze_From_Recording', {})]
g_pool.plugins = Plugin_List(
g_pool, session_settings.get('loaded_plugins', default_plugins))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__
})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# grab new frame
if g_pool.capture.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = g_pool.capture.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.capture.play = False
logger.warning("end of video")
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(
g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(
g_pool.pupil_positions_by_frame[frame.index])
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture._recent_frame = frame
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
unused_elements = g_pool.gui.update()
for b in unused_elements.buttons:
button, action, mods = b
pos = glfw.glfwGetCursorPos(main_window)
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in unused_elements.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in unused_elements.chars:
for p in g_pool.plugins:
p.on_char(char_)
glfw.glfwSwapBuffers(main_window)
# present frames at appropriate speed
g_pool.capture.wait(frame)
glfw.glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = str(g_pool.version)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.capture.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except:
import traceback
trace = traceback.format_exc()
logger.error('Process Player crashed with trace:\n{}'.format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
sleep(1.0)