def append_section_menu(self, sec):
section_menu = ui.Growing_Menu('Section Settings')
section_menu.color = cygl_utils.RGBA(*sec['color'])
def make_calibrate_fn(sec):
def calibrate():
self.calibrate_section(sec)
return calibrate
def make_remove_fn(sec):
def remove():
del self.menu[self.sections.index(sec) - len(self.sections)]
del self.sections[self.sections.index(sec)]
self.correlate_and_publish()
return remove
def set_trim_fn(button, sec, key):
def trim(format_only=False):
if format_only:
left_idx, right_idx = sec[key]
else:
right_idx = self.g_pool.seek_control.trim_right
left_idx = self.g_pool.seek_control.trim_left
sec[key] = left_idx, right_idx
time_fmt = key.replace('_', ' ').split(' ')[0].title() + ': '
min_ts = self.g_pool.timestamps[0]
for idx in (left_idx, right_idx):
ts = self.g_pool.timestamps[idx] - min_ts
minutes = ts // 60
seconds = ts - (minutes * 60.)
time_fmt += ' {:02.0f}:{:02.0f} -'.format(
abs(minutes), seconds)
button.outer_label = time_fmt[:-2] # remove final ' -'
button.function = trim
section_menu.append(ui.Text_Input('label', sec, label='Label'))
section_menu.append(
ui.Selector('calibration_method',
sec,
label="Calibration Method",
labels=['Circle Marker', 'Natural Features'],
selection=['circle_marker', 'natural_features']))
section_menu.append(
ui.Selector('mapping_method',
sec,
label='Calibration Mode',
selection=['2d', '3d']))
section_menu.append(
ui.Text_Input('status',
sec,
label='Calibration Status',
setter=lambda _: _))
section_menu.append(
ui.Info_Text(
'This section is calibrated using reference markers found in a user set range "Calibration". The calibration is used to map pupil to gaze positions within a user set range "Mapping". Drag trim marks in the timeline to set a range and apply it.'
))
calib_range_button = ui.Button('Set from trim marks', None)
set_trim_fn(calib_range_button, sec, 'calibration_range')
calib_range_button.function(format_only=True) # set initial label
section_menu.append(calib_range_button)
mapping_range_button = ui.Button('Set from trim marks', None)
set_trim_fn(mapping_range_button, sec, 'mapping_range')
mapping_range_button.function(format_only=True) # set initial label
section_menu.append(mapping_range_button)
section_menu.append(ui.Button('Recalibrate', make_calibrate_fn(sec)))
section_menu.append(ui.Button('Remove section', make_remove_fn(sec)))
# manual gaze correction menu
offset_menu = ui.Growing_Menu('Manual Correction')
offset_menu.append(
ui.Info_Text('The manual correction feature allows you to apply' +
' a fixed offset to your gaze data.'))
offset_menu.append(
ui.Slider('x_offset', sec, min=-.5, step=0.01, max=.5))
offset_menu.append(
ui.Slider('y_offset', sec, min=-.5, step=0.01, max=.5))
offset_menu.collapsed = True
section_menu.append(offset_menu)
self.menu.append(section_menu)
def world(timebase,eyes_are_alive,ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version):
"""Reads world video and runs plugins.
Creates a window, gl context.
Grabs images from a capture.
Maps pupil to gaze data
Can run various plug-ins.
Reacts to notifications:
``set_detection_mapping_mode``
``eye_process.started``
``start_plugin``
Emits notifications:
``eye_process.should_start``
``eye_process.should_stop``
``set_detection_mapping_mode``
``world_process.started``
``world_process.stopped``
``recording.should_stop``: Emits on camera failure
``launcher_process.should_stop``
Emits data:
``gaze``: Gaze data from current gaze mapping plugin.``
``*``: any other plugin generated data in the events that it not [dt,pupil,gaze].
"""
# We defer the imports because of multiprocessing.
# Otherwise the world process each process also loads the other imports.
# This is not harmful but unnecessary.
#general imports
from time import time,sleep
import numpy as np
import logging
#networking
import zmq
import zmq_tools
#zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url)
gaze_pub = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url)
pupil_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('pupil',))
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=('notify',))
#log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#display
import glfw
from pyglui import ui,graph,cygl,__version__ as pyglui_version
assert pyglui_version >= '1.0'
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen,make_coord_system_pixel_based,make_coord_system_norm_based,glFlush,is_window_visible
#monitoring
import psutil
# helpers/utils
from version_utils import VersionFormat
from file_methods import Persistent_Dict
from methods import normalize, denormalize, delta_t, get_system_info
from uvc import get_time_monotonic
logger.info('Application Version: %s'%version)
logger.info('System Info: %s'%get_system_info())
# video sources
from video_capture import InitialisationError,StreamError, Fake_Source, EndofVideoFileError, source_classes, manager_classes
source_by_name = {src.class_name():src for src in source_classes}
import audio
#trigger pupil detector cpp build:
import pupil_detectors
del pupil_detectors
# Plug-ins
from plugin import Plugin,Plugin_List,import_runtime_plugins
from calibration_routines import calibration_plugins, gaze_mapping_plugins
from fixation_detector import Fixation_Detector_3D
from recorder import Recorder
from show_calibration import Show_Calibration
from display_recent_gaze import Display_Recent_Gaze
from time_sync import Time_Sync
from pupil_remote import Pupil_Remote
from pupil_groups import Pupil_Groups
from surface_tracker import Surface_Tracker
from log_display import Log_Display
from annotations import Annotation_Capture
from log_history import Log_History
from frame_publisher import Frame_Publisher
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (0,0)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (8,31)
else:
scroll_factor = 1.0
window_position_default = (0,0)
#g_pool holds variables for this process they are accesible to all plugins
g_pool = Global_Container()
g_pool.app = 'capture'
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.timebase = timebase
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.eyes_are_alive = eyes_are_alive
def get_timestamp():
return get_time_monotonic()-g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
#manage plugins
runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir,'plugins'))
user_launchable_plugins = [Pupil_Groups,Frame_Publisher,Show_Calibration,Pupil_Remote,Time_Sync,Surface_Tracker,Annotation_Capture,Log_History,Fixation_Detector_3D]+runtime_plugins
system_plugins = [Log_Display,Display_Recent_Gaze,Recorder]
plugin_by_index = system_plugins+user_launchable_plugins+calibration_plugins+gaze_mapping_plugins+manager_classes
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index,plugin_by_index))
default_plugins = [('UVC_Manager',{}),('Log_Display',{}),('Dummy_Gaze_Mapper',{}),('Display_Recent_Gaze',{}), ('Screen_Marker_Calibration',{}),('Recorder',{}),('Pupil_Remote',{}),('Fixation_Detector_3D',{})]
# Callback functions
def on_resize(window,w, h):
if is_window_visible(window):
g_pool.gui.update_window(w,h)
g_pool.gui.collect_menus()
graph.adjust_size(w,h)
adjust_gl_view(w,h)
for p in g_pool.plugins:
p.on_window_resize(window,w,h)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_button(window,button, action, mods):
g_pool.gui.update_button(button,action,mods)
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
g_pool.gui.update_mouse(x,y)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world'))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
default_settings = {
'source_class_name': 'UVC_Source',
'preferred_names' : ["Pupil Cam1 ID2","Logitech Camera","(046d:081d)","C510","B525", "C525","C615","C920","C930e"],
'frame_size': (1280,720),
'frame_rate': 30
}
settings = session_settings.get('capture_settings', default_settings)
try:
cap = source_by_name[settings['source_class_name']](g_pool, **settings)
except (KeyError,InitialisationError) as e:
if isinstance(e,KeyError):
logger.warning('Incompatible capture setting encountered. Falling back to fake source.')
cap = Fake_Source(g_pool, **settings)
g_pool.iconified = False
g_pool.detection_mapping_mode = session_settings.get('detection_mapping_mode','3d')
g_pool.active_calibration_plugin = None
g_pool.active_gaze_mapping_plugin = None
g_pool.capture = cap
g_pool.capture_manager = None
audio.audio_mode = session_settings.get('audio_mode',audio.default_audio_mode)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def launch_eye_process(eye_id,delay=0):
n = {'subject':'eye_process.should_start.%s'%eye_id,'eye_id':eye_id,'delay':delay}
ipc_pub.notify(n)
def stop_eye_process(eye_id):
n = {'subject':'eye_process.should_stop','eye_id':eye_id}
ipc_pub.notify(n)
def start_stop_eye(eye_id,make_alive):
if make_alive:
launch_eye_process(eye_id)
else:
stop_eye_process(eye_id)
def set_detection_mapping_mode(new_mode):
n = {'subject':'set_detection_mapping_mode','mode':new_mode}
ipc_pub.notify(n)
def handle_notifications(n):
subject = n['subject']
if subject == 'set_detection_mapping_mode':
if n['mode'] == '2d':
if "Vector_Gaze_Mapper" in g_pool.active_gaze_mapping_plugin.class_name:
logger.warning("The gaze mapper is not supported in 2d mode. Please recalibrate.")
g_pool.plugins.add(plugin_by_name['Dummy_Gaze_Mapper'])
g_pool.detection_mapping_mode = n['mode']
elif subject == 'start_plugin':
g_pool.plugins.add(plugin_by_name[n['name']],args=n.get('args',{}) )
elif subject == 'eye_process.started':
n = {'subject':'set_detection_mapping_mode','mode':g_pool.detection_mapping_mode}
ipc_pub.notify(n)
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject':'meta.doc',
'actor':g_pool.app,
'doc':world.__doc__})
for p in g_pool.plugins:
if p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify:
ipc_pub.notify({
'subject':'meta.doc',
'actor': p.class_name,
'doc':p.on_notify.__doc__})
#window and gl setup
glfw.glfwInit()
width,height = session_settings.get('window_size',cap.frame_size)
main_window = glfw.glfwCreateWindow(width,height, "World")
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-350,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Selector('audio_mode',audio,selection=audio.audio_modes))
general_settings.append(ui.Selector('detection_mapping_mode',g_pool,label='detection & mapping mode',setter=set_detection_mapping_mode,selection=['2d','3d']))
general_settings.append(ui.Switch('eye0_process',label='Detect eye 0',setter=lambda alive: start_stop_eye(0,alive),getter=lambda: eyes_are_alive[0].value ))
general_settings.append(ui.Switch('eye1_process',label='Detect eye 1',setter=lambda alive: start_stop_eye(1,alive),getter=lambda: eyes_are_alive[1].value ))
selector_label = "Select to load"
labels = [p.__name__.replace('_',' ') for p in user_launchable_plugins]
user_launchable_plugins.insert(0, selector_label)
labels.insert(0, selector_label)
general_settings.append(ui.Selector('Open plugin',
selection = user_launchable_plugins,
labels = labels,
setter = open_plugin,
getter = lambda: selector_label))
general_settings.append(ui.Info_Text('Capture Version: %s'%g_pool.version))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar',(0,100),(120,-100))
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture.init_gui()
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(g_pool.quickbar)
#plugins that are loaded based on user settings from previous session
g_pool.plugins = Plugin_List(g_pool,plugin_by_name,session_settings.get('loaded_plugins',default_plugins))
#We add the calibration menu selector, after a calibration has been added:
g_pool.calibration_menu.insert(0,ui.Selector(
'active_calibration_plugin',
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection = calibration_plugins,
labels = [p.__name__.replace('_',' ') for p in calibration_plugins],
setter= open_plugin,label='Method'
))
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(0,ui.Selector(
'capture_manager',
setter = open_plugin,
getter = lambda: g_pool.capture_manager.__class__,
selection = manager_classes,
labels = [b.gui_name for b in manager_classes],
label = 'Manager'
))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
# refresh speed settings
glfw.glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil0_graph = graph.Bar_Graph(max_val=1.0)
pupil0_graph.pos = (260,130)
pupil0_graph.update_rate = 5
pupil0_graph.label = "id0 conf: %0.2f"
pupil1_graph = graph.Bar_Graph(max_val=1.0)
pupil1_graph.pos = (380,130)
pupil1_graph.update_rate = 5
pupil1_graph.label = "id1 conf: %0.2f"
pupil_graphs = pupil0_graph,pupil1_graph
if session_settings.get('eye1_process_alive',False):
launch_eye_process(1,delay=0.6)
if session_settings.get('eye0_process_alive',True):
launch_eye_process(0,delay=0.3)
ipc_pub.notify({'subject':'world_process.started'})
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t,n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
g_pool.capture.on_notify(n)
for p in g_pool.plugins:
p.on_notify(n)
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except StreamError as e:
prev_settings = g_pool.capture.settings
g_pool.capture.deinit_gui()
g_pool.capture.cleanup()
g_pool.capture = None
prev_settings['info_text'] ="'%s' disconnected."%prev_settings['name']
g_pool.capture = Fake_Source(g_pool,**prev_settings)
g_pool.capture.init_gui()
ipc_pub.notify({'subject':'recording.should_stop'})
logger.error("Error getting frame. Falling back to Fake source.")
logger.debug("Caught error: %s"%e)
sleep(.2)
continue
except EndofVideoFileError:
logger.warning("Video file is done. Rewinding")
g_pool.capture.seek_to_frame(0)
continue
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
recent_pupil_data = []
recent_gaze_data = []
while pupil_sub.new_data:
t,p = pupil_sub.recv()
pupil_graphs[p['id']].add(p['confidence'])
recent_pupil_data.append(p)
new_gaze_data = g_pool.active_gaze_mapping_plugin.on_pupil_datum(p)
for g in new_gaze_data:
gaze_pub.send('gaze',g)
recent_gaze_data += new_gaze_data
events['pupil_positions'] = recent_pupil_data
events['gaze_positions'] = recent_gaze_data
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
#send new events to ipc:
del events['pupil_positions'] #already on the wire
del events['gaze_positions'] #send earlier in this loop
del events['dt'] #no need to send this
for topic,data in events.iteritems():
assert(isinstance(data, (list, tuple)))
for d in data:
ipc_pub.send(topic, d)
# render camera image
glfw.glfwMakeContextCurrent(main_window)
if is_window_visible(main_window):
g_pool.image_tex.update_from_frame(frame)
glFlush()
make_coord_system_norm_based()
g_pool.image_tex.draw()
make_coord_system_pixel_based((frame.height,frame.width,3))
# render visual feedback from loaded plugins
if is_window_visible(main_window):
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil0_graph.draw()
pupil1_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['eye0_process_alive'] = eyes_are_alive[0].value
session_settings['eye1_process_alive'] = eyes_are_alive[1].value
session_settings['detection_mapping_mode'] = g_pool.detection_mapping_mode
session_settings['audio_mode'] = audio.audio_mode
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture.deinit_gui()
g_pool.capture.cleanup()
#shut down eye processes:
stop_eye_process(0)
stop_eye_process(1)
logger.info("Process shutting down.")
ipc_pub.notify({'subject':'world_process.stopped'})
#shut down launcher
n = {'subject':'launcher_process.should_stop'}
ipc_pub.notify(n)
zmq_ctx.destroy()
def init_ui(self):
polyline_style_thickness_slider = ui.Slider(
"thickness",
self.polyline_style_controller,
min=self.polyline_style_controller.thickness_min,
max=self.polyline_style_controller.thickness_max,
step=self.polyline_style_controller.thickness_step,
label="Line thickness",
)
polyline_style_color_info_text = ui.Info_Text(
"Set RGB color component values.")
polyline_style_color_r_slider = ui.Slider(
"r",
self.polyline_style_controller,
min=self.polyline_style_controller.rgba_min,
max=self.polyline_style_controller.rgba_max,
step=self.polyline_style_controller.rgba_step,
label="Red",
)
polyline_style_color_g_slider = ui.Slider(
"g",
self.polyline_style_controller,
min=self.polyline_style_controller.rgba_min,
max=self.polyline_style_controller.rgba_max,
step=self.polyline_style_controller.rgba_step,
label="Green",
)
polyline_style_color_b_slider = ui.Slider(
"b",
self.polyline_style_controller,
min=self.polyline_style_controller.rgba_min,
max=self.polyline_style_controller.rgba_max,
step=self.polyline_style_controller.rgba_step,
label="Blue",
)
scan_path_timeframe_range = ui.Slider(
"timeframe",
self.scan_path_controller,
min=self.scan_path_controller.min_timeframe,
max=self.scan_path_controller.max_timeframe,
step=self.scan_path_controller.timeframe_step,
label="Duration",
)
scan_path_doc = ui.Info_Text(
"Duration of past gaze to include in polyline.")
scan_path_status = ui.Info_Text("")
polyline_style_color_menu = ui.Growing_Menu("Color")
polyline_style_color_menu.collapsed = True
polyline_style_color_menu.append(polyline_style_color_info_text)
polyline_style_color_menu.append(polyline_style_color_r_slider)
polyline_style_color_menu.append(polyline_style_color_g_slider)
polyline_style_color_menu.append(polyline_style_color_b_slider)
scan_path_menu = ui.Growing_Menu("Gaze History")
scan_path_menu.collapsed = False
scan_path_menu.append(scan_path_doc)
scan_path_menu.append(scan_path_timeframe_range)
scan_path_menu.append(scan_path_status)
self.add_menu()
self.menu.label = "Gaze Polyline"
self.menu.append(polyline_style_thickness_slider)
self.menu.append(polyline_style_color_menu)
self.menu.append(scan_path_menu)
self.scan_path_timeframe_range = scan_path_timeframe_range
self.scan_path_status = scan_path_status
self._update_scan_path_ui()
def init_ui(self):
self.add_menu()
self.menu.label = 'Fixation Detector'
def set_max_dispersion(new_value):
self.max_dispersion = new_value
self.notify_all({
'subject': 'fixation_detector.should_recalculate',
'delay': 1.
})
def set_min_duration(new_value):
self.min_duration = min(new_value, self.max_duration)
self.notify_all({
'subject': 'fixation_detector.should_recalculate',
'delay': 1.
})
def set_max_duration(new_value):
self.max_duration = max(new_value, self.min_duration)
self.notify_all({
'subject': 'fixation_detector.should_recalculate',
'delay': 1.
})
def jump_next_fixation(_):
cur_idx = self.last_frame_idx
all_idc = [f['mid_frame_index'] for f in self.g_pool.fixations]
if not all_idc:
logger.warning('No fixations available')
return
# wrap-around index
tar_fix = bisect_right(all_idc, cur_idx) % len(all_idc)
self.notify_all({
'subject':
'seek_control.should_seek',
'index':
int(self.g_pool.fixations[tar_fix]['mid_frame_index'])
})
def jump_prev_fixation(_):
cur_idx = self.last_frame_idx
all_idc = [f['mid_frame_index'] for f in self.g_pool.fixations]
if not all_idc:
logger.warning('No fixations available')
return
# wrap-around index
tar_fix = (bisect_left(all_idc, cur_idx) - 1) % len(all_idc)
self.notify_all({
'subject':
'seek_control.should_seek',
'index':
int(self.g_pool.fixations[tar_fix]['mid_frame_index'])
})
for help_block in self.__doc__.split('\n\n'):
help_str = help_block.replace('\n', ' ').replace(' ', '').strip()
self.menu.append(ui.Info_Text(help_str))
self.menu.append(
ui.Info_Text(
"Press the export button or type 'e' to start the export."))
self.menu.append(
ui.Slider('max_dispersion',
self,
min=0.01,
step=0.1,
max=5.,
label='Maximum Dispersion [degrees]',
setter=set_max_dispersion))
self.menu.append(
ui.Slider('min_duration',
self,
min=10,
step=10,
max=4000,
label='Minimum Duration [milliseconds]',
setter=set_min_duration))
self.menu.append(
ui.Slider('max_duration',
self,
min=10,
step=10,
max=4000,
label='Maximum Duration [milliseconds]',
setter=set_max_duration))
self.menu.append(
ui.Text_Input('status',
self,
label='Detection progress:',
setter=lambda x: None))
self.menu.append(
ui.Switch('show_fixations', self, label='Show fixations'))
self.current_fixation_details = ui.Info_Text('')
self.menu.append(self.current_fixation_details)
self.next_fix_button = ui.Thumb('jump_next_fixation',
setter=jump_next_fixation,
getter=lambda: False,
label=chr(0xe044),
hotkey='f',
label_font='pupil_icons')
self.next_fix_button.status_text = 'Next Fixation'
self.g_pool.quickbar.append(self.next_fix_button)
self.prev_fix_button = ui.Thumb('jump_prev_fixation',
setter=jump_prev_fixation,
getter=lambda: False,
label=chr(0xe045),
hotkey='F',
label_font='pupil_icons')
self.prev_fix_button.status_text = 'Previous Fixation'
self.g_pool.quickbar.append(self.prev_fix_button)
def update_menu(self):
del self.menu[:]
from pyglui import ui
ui_elements = []
# lets define some helper functions:
def gui_load_defaults():
for c in self.uvc_capture.controls:
try:
c.value = c.def_val
except:
pass
def gui_update_from_device():
for c in self.uvc_capture.controls:
c.refresh()
def set_frame_size(new_size):
self.frame_size = new_size
def set_frame_rate(new_rate):
self.frame_rate = new_rate
self.update_menu()
if self.uvc_capture is None:
ui_elements.append(ui.Info_Text('Capture initialization failed.'))
self.menu.extend(ui_elements)
return
ui_elements.append(ui.Info_Text('{} Controls'.format(self.name)))
sensor_control = ui.Growing_Menu(label='Sensor Settings')
sensor_control.append(
ui.Info_Text(
"Do not change these during calibration or recording!"))
sensor_control.collapsed = False
image_processing = ui.Growing_Menu(label='Image Post Processing')
image_processing.collapsed = True
sensor_control.append(
ui.Selector('frame_size',
self,
setter=set_frame_size,
selection=self.uvc_capture.frame_sizes,
label='Resolution'))
def frame_rate_getter():
return (self.uvc_capture.frame_rates,
[str(fr) for fr in self.uvc_capture.frame_rates])
sensor_control.append(
ui.Selector('frame_rate',
self,
selection_getter=frame_rate_getter,
setter=set_frame_rate,
label='Frame rate'))
if ("Pupil Cam2" in self.uvc_capture.name):
special_settings = {200: 28, 180: 31}
def set_exposure_mode(exposure_mode):
self.exposure_mode = exposure_mode
if self.exposure_mode == "auto":
self.preferred_exposure_time = Exposure_Time(
max_ET=special_settings.get(self.frame_rate, 32),
frame_rate=self.frame_rate,
mode=self.exposure_mode)
else:
self.preferred_exposure_time = None
logger.info(
"Exposure mode for camera {0} is now set to {1} mode".
format(self.uvc_capture.name, exposure_mode))
self.update_menu()
def exposure_mode_getter():
return ["manual", "auto"], ["manual mode", "auto mode"]
sensor_control.append(
ui.Selector('exposure_mode',
self,
setter=set_exposure_mode,
selection_getter=exposure_mode_getter,
selection=self.exposure_mode,
label="Exposure Mode"))
sensor_control.append(
ui.Slider('exposure_time',
self,
label='Absolute Exposure Time',
min=1,
max=special_settings.get(self.frame_rate, 32),
step=1))
if self.exposure_mode == "auto":
sensor_control[-1].read_only = True
if ("Pupil Cam" in self.uvc_capture.name):
blacklist = [
'Auto Focus', 'Absolute Focus', 'Absolute Iris ',
'Scanning Mode ', 'Zoom absolute control', 'Pan control',
'Tilt control', 'Roll absolute control',
'Privacy Shutter control'
]
else:
blacklist = []
if ("Pupil Cam2" in self.uvc_capture.name):
blacklist += [
'Auto Exposure Mode', 'Auto Exposure Priority',
'Absolute Exposure Time'
]
for control in self.uvc_capture.controls:
c = None
ctl_name = control.display_name
if ctl_name in blacklist:
continue
# now we add controls
if control.d_type == bool:
c = ui.Switch('value',
control,
label=ctl_name,
on_val=control.max_val,
off_val=control.min_val)
elif control.d_type == int:
c = ui.Slider('value',
control,
label=ctl_name,
min=control.min_val,
max=control.max_val,
step=control.step)
elif type(control.d_type) == dict:
selection = [value for name, value in control.d_type.items()]
labels = [name for name, value in control.d_type.items()]
c = ui.Selector('value',
control,
label=ctl_name,
selection=selection,
labels=labels)
else:
pass
# if control['disabled']:
# c.read_only = True
# if ctl_name == 'Exposure, Auto Priority':
# # the controll should always be off. we set it to 0 on init (see above)
# c.read_only = True
if c is not None:
if control.unit == 'processing_unit':
image_processing.append(c)
else:
sensor_control.append(c)
ui_elements.append(sensor_control)
if image_processing.elements:
ui_elements.append(image_processing)
ui_elements.append(ui.Button("refresh", gui_update_from_device))
if ("Pupil Cam2" in self.uvc_capture.name):
def set_check_stripes(check_stripes):
self.check_stripes = check_stripes
if self.check_stripes:
self.checkframestripes = Check_Frame_Stripes()
logger.info("Check Stripes for camera {} is now on".format(
self.uvc_capture.name))
else:
self.checkframestripes = None
logger.info(
"Check Stripes for camera {} is now off".format(
self.uvc_capture.name))
ui_elements.append(
ui.Switch('check_stripes',
self,
setter=set_check_stripes,
label="Check Stripes"))
self.menu.extend(ui_elements)
def add_controls_to_menu(self, menu, controls):
from pyglui import ui
# closure factory
def make_value_change_fn(ctrl_id):
def initiate_value_change(val):
logger.debug("{}: {} >> {}".format(self.sensor, ctrl_id, val))
self.sensor.set_control_value(ctrl_id, val)
return initiate_value_change
for ctrl_id, ctrl_dict in controls:
try:
dtype = ctrl_dict["dtype"]
ctrl_ui = None
if dtype == "string":
ctrl_ui = ui.Text_Input(
"value",
ctrl_dict,
label=ctrl_dict["caption"],
setter=make_value_change_fn(ctrl_id),
)
elif dtype == "integer" or dtype == "float":
convert_fn = int if dtype == "integer" else float
ctrl_ui = ui.Slider(
"value",
ctrl_dict,
label=ctrl_dict["caption"],
min=convert_fn(ctrl_dict.get("min", 0)),
max=convert_fn(ctrl_dict.get("max", 100)),
step=convert_fn(ctrl_dict.get("res", 0.0)),
setter=make_value_change_fn(ctrl_id),
)
elif dtype == "bool":
ctrl_ui = ui.Switch(
"value",
ctrl_dict,
label=ctrl_dict["caption"],
on_val=ctrl_dict.get("max", True),
off_val=ctrl_dict.get("min", False),
setter=make_value_change_fn(ctrl_id),
)
elif dtype == "strmapping" or dtype == "intmapping":
desc_list = ctrl_dict["map"]
labels = [desc["caption"] for desc in desc_list]
selection = [desc["value"] for desc in desc_list]
ctrl_ui = ui.Selector(
"value",
ctrl_dict,
label=ctrl_dict["caption"],
labels=labels,
selection=selection,
setter=make_value_change_fn(ctrl_id),
)
if ctrl_ui:
ctrl_ui.read_only = ctrl_dict.get("readonly", False)
self.control_id_ui_mapping[ctrl_id] = ctrl_ui
menu.append(ctrl_ui)
else:
logger.error("Did not generate UI for {}".format(ctrl_id))
except:
logger.error("Exception for control:\n{}".format(ctrl_dict))
import traceback as tb
tb.print_exc()
if len(menu) == 0:
menu.append(ui.Info_Text("No {} settings found".format(
menu.label)))
return menu
def update_menu(self):
del self.menu[:]
from pyglui import ui
ui_elements = []
# lets define some helper functions:
def gui_load_defaults():
for c in self.uvc_capture.controls:
try:
c.value = c.def_val
except:
pass
def gui_update_from_device():
for c in self.uvc_capture.controls:
c.refresh()
def set_frame_size(new_size):
self.frame_size = new_size
if self.uvc_capture is None:
ui_elements.append(ui.Info_Text('Capture initialization failed.'))
self.menu.extend(ui_elements)
return
ui_elements.append(ui.Info_Text('{} Controls'.format(self.name)))
sensor_control = ui.Growing_Menu(label='Sensor Settings')
sensor_control.append(
ui.Info_Text(
"Do not change these during calibration or recording!"))
sensor_control.collapsed = False
image_processing = ui.Growing_Menu(label='Image Post Processing')
image_processing.collapsed = True
sensor_control.append(
ui.Selector('frame_size',
self,
setter=set_frame_size,
selection=self.uvc_capture.frame_sizes,
label='Resolution'))
def frame_rate_getter():
return (self.uvc_capture.frame_rates,
[str(fr) for fr in self.uvc_capture.frame_rates])
sensor_control.append(
ui.Selector('frame_rate',
self,
selection_getter=frame_rate_getter,
label='Frame rate'))
if ("Pupil Cam" in self.uvc_capture.name):
blacklist = [
'Auto Focus', 'Absolute Focus', 'Absolute Iris ',
'Scanning Mode ', 'Zoom absolute control', 'Pan control',
'Tilt control', 'Roll absolute control',
'Privacy Shutter control'
]
else:
blacklist = []
for control in self.uvc_capture.controls:
c = None
ctl_name = control.display_name
if ctl_name in blacklist:
continue
# now we add controls
if control.d_type == bool:
c = ui.Switch('value',
control,
label=ctl_name,
on_val=control.max_val,
off_val=control.min_val)
elif control.d_type == int:
c = ui.Slider('value',
control,
label=ctl_name,
min=control.min_val,
max=control.max_val,
step=control.step)
elif type(control.d_type) == dict:
selection = [value for name, value in control.d_type.items()]
labels = [name for name, value in control.d_type.items()]
c = ui.Selector('value',
control,
label=ctl_name,
selection=selection,
labels=labels)
else:
pass
# if control['disabled']:
# c.read_only = True
# if ctl_name == 'Exposure, Auto Priority':
# # the controll should always be off. we set it to 0 on init (see above)
# c.read_only = True
if c is not None:
if control.unit == 'processing_unit':
image_processing.append(c)
else:
sensor_control.append(c)
ui_elements.append(sensor_control)
if image_processing.elements:
ui_elements.append(image_processing)
ui_elements.append(ui.Button("refresh", gui_update_from_device))
ui_elements.append(ui.Button("load defaults", gui_load_defaults))
self.menu.extend(ui_elements)
def init_gui(self):
self.monitor_idx = 0
self.monitor_names = [glfwGetMonitorName(m) for m in glfwGetMonitors()]
#primary_monitor = glfwGetPrimaryMonitor()
self.info = ui.Info_Text(
"Calibrate gaze parameters using a screen based animation.")
self.g_pool.calibration_menu.append(self.info)
self.menu = ui.Growing_Menu('Controls')
self.menu.configuration = self.menu_conf
self.g_pool.calibration_menu.append(self.menu)
self.menu.append(
ui.Selector('monitor_idx',
self,
selection=range(len(self.monitor_names)),
labels=self.monitor_names,
label='Monitor'))
self.menu.append(ui.Switch('fullscreen', self, label='Use fullscreen'))
self.menu.append(
ui.Slider('marker_scale',
self,
step=0.1,
min=0.5,
max=2.0,
label='Pattern scale'))
submenu = ui.Growing_Menu('Advanced')
submenu.collapsed = True
self.menu.append(submenu)
submenu.append(
ui.Slider('sample_duration',
self,
step=1,
min=10,
max=100,
label='Sample duration',
setter=self.update_sample_duration))
submenu.append(ui.Switch('show_edges', self, label='show edges'))
submenu.append(
ui.Slider('area_threshold',
self,
step=1,
min=5,
max=50,
label='Area threshold'))
submenu.append(
ui.Slider('dist_threshold',
self,
step=.5,
min=1,
max=20,
label='Eccetricity threshold'))
self.button = ui.Thumb('active',
self,
setter=self.toggle,
label='Calibrate',
hotkey='c')
self.button.on_color[:] = (.3, .2, 1., .9)
self.g_pool.quickbar.insert(0, self.button)
def init_ui(self):
self.add_menu()
self.menu.label = "Light Points"
self.menu.append(ui.Slider("falloff", self, min=1, step=1, max=1000))
def eye(timebase, is_alive_flag, ipc_pub_url, ipc_sub_url,ipc_push_url, user_dir, version, eye_id,overwrite_cap_settings=None):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx,ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx,ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,ipc_sub_url,topics=("notify",))
with Is_Alive_Manager(is_alive_flag,ipc_socket,eye_id):
#logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx,ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based,is_window_visible
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from uvc import get_time_monotonic, StreamError
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from av_writer import JPEG_Writer,AV_Writer
from video_capture import InitialisationError,StreamError, Fake_Source,EndofVideoFileError, source_classes, manager_classes
source_by_name = {src.class_name():src for src in source_classes}
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic()-g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window,w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
g_pool.on_frame_size_change = lambda new_size: None
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
capture_manager_settings = session_settings.get(
'capture_manager_settings', ('UVC_Manager',{}))
if eye_id == 0:
cap_src = ["Pupil Cam1 ID0","HD-6000","Integrated Camera","HD USB Camera","USB 2.0 Camera"]
else:
cap_src = ["Pupil Cam1 ID1","HD-6000","Integrated Camera"]
# Initialize capture
default_settings = {
'source_class_name': 'UVC_Source',
'preferred_names' : cap_src,
'frame_size': (640,480),
'frame_rate': 90
}
settings = overwrite_cap_settings or session_settings.get('capture_settings', default_settings)
try:
cap = source_by_name[settings['source_class_name']](g_pool, **settings)
except (KeyError,InitialisationError) as e:
if isinstance(e,KeyError):
logger.warning('Incompatible capture setting encountered. Falling back to fake source.')
cap = Fake_Source(g_pool, **settings)
g_pool.iconified = False
g_pool.capture = cap
g_pool.capture_manager = None
g_pool.flip = session_settings.get('flip',False)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1],g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get('roi')
if roi_user_settings and roi_user_settings[-1] == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1],new_size[0]))
g_pool.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s"%eye_id
width,height = session_settings.get('window_size',g_pool.capture.frame_size)
main_window = glfw.glfwCreateWindow(width,height, title, None, None)
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
glfw.glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
general_settings.append(detector_selector)
g_pool.capture_selector_menu = ui.Growing_Menu('Capture Selection')
g_pool.capture_source_menu = ui.Growing_Menu('Capture Source')
g_pool.capture.init_gui()
g_pool.sidebar.append(general_settings)
g_pool.sidebar.append(g_pool.capture_selector_menu)
g_pool.sidebar.append(g_pool.capture_source_menu)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__:c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](g_pool,**manager_settings)
g_pool.capture_manager.init_gui()
def open_manager(manager_class):
g_pool.capture_manager.cleanup()
g_pool.capture_manager = manager_class(g_pool)
g_pool.capture_manager.init_gui()
#We add the capture selection menu, after a manager has been added:
g_pool.capture_selector_menu.insert(0,ui.Selector(
'capture_manager',g_pool,
setter = open_manager,
getter = lambda: g_pool.capture_manager.__class__,
selection = manager_classes,
labels = [b.gui_name for b in manager_classes],
label = 'Manager'
))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
should_publish_frames = False
frame_publish_format = 'jpeg'
#create a timer to control window update frequency
window_update_timer = timer(1/60.)
def window_should_update():
return next(window_update_timer)
logger.warning('Process started.')
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t,notification = notify_sub.recv()
subject = notification['subject']
if subject == 'eye_process.should_stop':
if notification['eye_id'] == eye_id:
break
elif subject == 'set_detection_mapping_mode':
if notification['mode'] == '3d':
if not isinstance(g_pool.pupil_detector,Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
if not isinstance(g_pool.pupil_detector,Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
elif subject == 'recording.started':
if notification['record_eye']:
record_path = notification['rec_path']
raw_mode = notification['compression']
logger.info("Will save eye video to: %s"%record_path)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = JPEG_Writer(video_path,g_pool.capture.frame_rate)
else:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = AV_Writer(video_path,g_pool.capture.frame_rate)
timestamps = []
elif subject == 'recording.stopped':
if writer:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
elif subject.startswith('meta.should_doc'):
ipc_socket.notify({
'subject':'meta.doc',
'actor':'eye%i'%eye_id,
'doc':eye.__doc__
})
elif subject.startswith('frame_publishing.started'):
should_publish_frames = True
frame_publish_format = notification.get('format','jpeg')
elif subject.startswith('frame_publishing.stopped'):
should_publish_frames = False
frame_publish_format = 'jpeg'
else:
g_pool.capture_manager.on_notify(notification)
# Get an image from the grabber
try:
frame = g_pool.capture.get_frame()
except StreamError as e:
logger.error("Error getting frame. Stopping eye process.")
logger.debug("Caught error: %s"%e)
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
g_pool.capture.seek_to_frame(0)
frame = g_pool.capture.get_frame()
g_pool.capture_manager.update(frame, {})
if should_publish_frames and frame.jpeg_buffer:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
pupil_socket.send('frame.eye.%s'%eye_id,{
'width': frame.width,
'height': frame.width,
'index': frame.index,
'timestamp': frame.timestamp,
'format': frame_publish_format,
'__raw_data__': [data]
})
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
pupil_socket.send('pupil.%s'%eye_id,result)
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
g_pool.capture.gl_display()
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
(int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
int(eye_ball['angle']),0,360,8)
except ValueError as e:
pass
else:
draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )
if result['confidence'] >0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
int(result['ellipse']['angle']),0,360,15)
confidence = result['confidence'] * 0.7 #scale it a little
draw_polyline(pts,1,RGBA(1.,0,0,confidence))
draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['capture_manager_settings'] = g_pool.capture_manager.class_name, g_pool.capture_manager.get_init_dict()
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.capture.deinit_gui()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
logger.info("Process shutting down.")
def init_ui(self):
self.add_menu()
self.menu.label = "Fixation Detector"
def set_max_dispersion(new_value):
self.max_dispersion = new_value
self.notify_all({
"subject": "fixation_detector.should_recalculate",
"delay": 1.0
})
def set_min_duration(new_value):
self.min_duration = min(new_value, self.max_duration)
self.notify_all({
"subject": "fixation_detector.should_recalculate",
"delay": 1.0
})
def set_max_duration(new_value):
self.max_duration = max(new_value, self.min_duration)
self.notify_all({
"subject": "fixation_detector.should_recalculate",
"delay": 1.0
})
def jump_next_fixation(_):
cur_idx = self.last_frame_idx
all_idc = [f["mid_frame_index"] for f in self.g_pool.fixations]
if not all_idc:
logger.warning("No fixations available")
return
# wrap-around index
tar_fix = bisect_right(all_idc, cur_idx) % len(all_idc)
self.notify_all({
"subject":
"seek_control.should_seek",
"index":
int(self.g_pool.fixations[tar_fix]["mid_frame_index"]),
})
def jump_prev_fixation(_):
cur_idx = self.last_frame_idx
all_idc = [f["mid_frame_index"] for f in self.g_pool.fixations]
if not all_idc:
logger.warning("No fixations available")
return
# wrap-around index
tar_fix = (bisect_left(all_idc, cur_idx) - 1) % len(all_idc)
self.notify_all({
"subject":
"seek_control.should_seek",
"index":
int(self.g_pool.fixations[tar_fix]["mid_frame_index"]),
})
for help_block in self.__doc__.split("\n\n"):
help_str = help_block.replace("\n", " ").replace(" ", "").strip()
self.menu.append(ui.Info_Text(help_str))
self.menu.append(
ui.Info_Text(
"Press the export button or type 'e' to start the export."))
self.menu.append(
ui.Slider(
"max_dispersion",
self,
min=0.01,
step=0.1,
max=5.0,
label="Maximum Dispersion [degrees]",
setter=set_max_dispersion,
))
self.menu.append(
ui.Slider(
"min_duration",
self,
min=10,
step=10,
max=4000,
label="Minimum Duration [milliseconds]",
setter=set_min_duration,
))
self.menu.append(
ui.Slider(
"max_duration",
self,
min=10,
step=10,
max=4000,
label="Maximum Duration [milliseconds]",
setter=set_max_duration,
))
self.menu.append(
ui.Text_Input("status",
self,
label="Detection progress:",
setter=lambda x: None))
self.menu.append(
ui.Switch("show_fixations", self, label="Show fixations"))
self.current_fixation_details = ui.Info_Text("")
self.menu.append(self.current_fixation_details)
self.next_fix_button = ui.Thumb(
"jump_next_fixation",
setter=jump_next_fixation,
getter=lambda: False,
label=chr(0xE044),
hotkey="f",
label_font="pupil_icons",
)
self.next_fix_button.status_text = "Next Fixation"
self.g_pool.quickbar.append(self.next_fix_button)
self.prev_fix_button = ui.Thumb(
"jump_prev_fixation",
setter=jump_prev_fixation,
getter=lambda: False,
label=chr(0xE045),
hotkey="F",
label_font="pupil_icons",
)
self.prev_fix_button.status_text = "Previous Fixation"
self.g_pool.quickbar.append(self.prev_fix_button)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version):
# general imports
import logging
import errno
from glob import glob
from copy import deepcopy
from time import time
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('notify', ))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from file_methods import Persistent_Dict, load_object
import numpy as np
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
import gl_utils
# capture
from video_capture import File_Source, EndofVideoFileError, FileSeekError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
# monitoring
import psutil
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_bar import Seek_Bar
from trim_marks import Trim_Marks
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
# from manual_gaze_correction import Manual_Gaze_Correction
from batch_exporter import Batch_Exporter
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
assert pyglui_version >= '1.5'
runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
user_launchable_plugins = [
Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path,
Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter,
Batch_Exporter, Annotation_Player, Log_History, Marker_Auto_Trim_Marks,
Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording,
Offline_Calibration
] + runtime_plugins
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
if gl_utils.is_window_visible(window):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug(
"Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({
"subject": "player_drop_process.should_start",
"rec_dir": new_rec_dir
})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error(
"'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264',
'.mjpeg')
][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
timestamps = np.load(timestamps_path)
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
# Initialize capture
cap = File_Source(g_pool, video_path, timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if VersionFormat(session_settings.get("version", '0.0')) != app_version:
logger.info(
"Session setting are a different version of this app. I will not use those."
)
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence',
0.6)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(
g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps
] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps
] # populated by producer
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() + 1)
except (FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 1)
except (FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if cap.get_frame_index() >= cap.get_frame_count() - 5:
cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time() + .8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
export_dir = os.path.join(g_pool.rec_dir, 'exports',
'{}-{}'.format(*export_range))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(*export_range))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {
'subject': 'should_export',
'range': export_range,
'export_dir': export_dir
}
g_pool.ipc_pub.notify(notification)
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.main_menu = ui.Scrolling_Menu("Settings",
pos=(-350, 20),
size=(300, 560))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.main_menu.append(
ui.Selector('gui_user_scale',
g_pool,
setter=set_scale,
selection=[.8, .9, 1., 1.1, 1.2],
label='Interface Size'))
g_pool.main_menu.append(
ui.Info_Text('Player Version: {}'.format(g_pool.version)))
g_pool.main_menu.append(
ui.Info_Text('Capture Version: {}'.format(
meta_info['Capture Software Version'])))
g_pool.main_menu.append(
ui.Info_Text('Data Format Version: {}'.format(
meta_info['Data Format Version'])))
g_pool.main_menu.append(
ui.Slider('min_data_confidence',
g_pool,
setter=set_data_confidence,
step=.05,
min=0.0,
max=1.0,
label='Confidence threshold'))
g_pool.main_menu.append(ui.Info_Text('Open plugins'))
selector_label = "Select to load"
def append_selector(label, plugins):
plugins.sort(key=lambda p: p.__name__)
plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
g_pool.main_menu.append(
ui.Selector(label,
selection=[selector_label] + plugins,
labels=[selector_label] + plugin_labels,
setter=open_plugin,
getter=lambda: selector_label))
base_plugins = [
Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
]
base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
launchable = user_launchable_plugins.copy()
for base_class, label in zip(base_plugins, base_labels):
member_plugins = []
for p in user_launchable_plugins:
if issubclass(p, base_class):
member_plugins.append(p)
launchable.remove(p)
append_selector(label, member_plugins)
# launchable only contains plugins that could not be assigned to any of the above categories
append_selector('Other', launchable)
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=chr(0xf04b),
setter=toggle_play,
hotkey=glfw.GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xf04e),
getter=lambda: False,
setter=next_frame,
hotkey=glfw.GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xf04a),
getter=lambda: False,
setter=prev_frame,
hotkey=glfw.GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=chr(0xf063),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([
g_pool.play_button, g_pool.forward_button, g_pool.backward_button,
g_pool.export_button
])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}),
('Vis_Polyline', {}), ('Vis_Circle', {}),
('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}),
('Gaze_From_Recording', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = None
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(plugin_by_name[n['name']],
args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__
})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(
g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(
g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts and ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
else:
ts = new_frame.timestamp
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfw.glfwMakeContextCurrent(main_window)
gl_utils.make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
gl_utils.make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
unused_buttons = g_pool.gui.update()
for b in unused_buttons:
button, action, mods = b
pos = glfw.glfwGetCursorPos(main_window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
# present frames at appropriate speed
cap.wait(frame)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
def init_gui(self):
# initialize the menu
def set_sequence_frames(seq_frame_num):
self.seq_frame_num = seq_frame_num
def on_aoi_encode():
self.aoi_provider.rw_aoi_binary()
logger.info("All AOI's are hashed by hex-32")
self.menu = ui.Scrolling_Menu('Scan Path')
self.menu.append(ui.Button('Close', self.unset_alive))
#Scan-path properties
scanpath_menu = ui.Growing_Menu('ScanPath Properties')
scanpath_menu.append(ui.Slider('timeframe', self, min=0, step=0.1, max=5, label="Timeframe in sec"))
scanpath_menu.append(ui.Button('Encode AOI manually', on_aoi_encode))
scanpath_menu.append(ui.Slider('seq_frame_num', self, min=1, step=1, max=20, label='Number of frames in a fixation sequence',
setter=set_sequence_frames))
scanpath_menu.append(ui.Button('Save Fixations and Timestamps', self.on_scanpath_build))
#scanpath_menu.append(ui.Button('Sample frames by similarity', self.on_sample_frames_by_similarity))
self.menu.append(scanpath_menu)
def set_h_fov(new_fov):
self.h_fov = new_fov
self.pix_per_degree = float(self.g_pool.capture.frame_size[0]) / new_fov
self.notify_all({'subject': 'fixations_should_recalculate', 'delay': 1.})
def set_v_fov(new_fov):
self.v_fov = new_fov
self.pix_per_degree = float(self.g_pool.capture.frame_size[1]) / new_fov
self.notify_all({'subject': 'fixations_should_recalculate', 'delay': 1.})
def set_duration(new_value):
self.min_duration = new_value
self.notify_all({'subject': 'fixations_should_recalculate', 'delay': 1.})
def set_dispersion(new_value):
self.max_dispersion = new_value
self.notify_all({'subject': 'fixations_should_recalculate', 'delay': 1.})
def jump_next_fixation(_):
ts = self.last_frame_ts
for f in self.fixations:
if f['timestamp'] > ts:
self.g_pool.capture.seek_to_frame(f['mid_frame_index'])
self.g_pool.new_seek = True
return
logger.error('could not seek to next fixation.')
fixation_menu = ui.Growing_Menu('Fixation Properties')
fixation_menu.append(ui.Info_Text("Fixation Properties"))
fixation_menu.append(ui.Info_Text("Press the export button or type 'e' to start the export."))
fixation_menu.append(ui.Slider('min_duration', self, min=0.0, step=0.05, max=1.0, label='Duration threshold',
setter=set_duration))
fixation_menu.append(ui.Slider('max_dispersion', self,
min=self.dispersion_slider_min,
step=self.dispersion_slider_stp,
max=self.dispersion_slider_max,
label='Dispersion threshold',
setter=set_dispersion))
fixation_menu.append(
ui.Slider('h_fov', self, min=5, step=1, max=180, label='Horizontal FOV of scene camera', setter=set_h_fov))
fixation_menu.append(
ui.Slider('v_fov', self, min=5, step=1, max=180, label='Vertical FOV of scene camera', setter=set_v_fov))
self.menu.append(fixation_menu)
self.g_pool.gui.append(self.menu)
def init_gui(self):
# initialize the menu
self.menu = ui.Scrolling_Menu('Gaze Circles on Contours')
# add menu to the window
self.g_pool.gui.append(self.menu)
self.menu.append(ui.Button('Close', self.unset_alive))
self.menu.append(ui.Info_Text('Circle Properties'))
self.menu.append(
ui.Slider('radius', self, min=1, step=1, max=100, label='Radius'))
self.menu.append(
ui.Slider('thickness',
self,
min=1,
step=1,
max=15,
label='Stroke width'))
self.menu.append(ui.Switch('fill', self, label='Fill'))
self.menu.append(ui.Info_Text('Detector Properties'))
self.menu.append(
ui.Slider('ellipse_size',
self,
min=0,
step=0.001,
max=4,
label='Ellipse Size'))
self.menu.append(
ui.Slider('epsilon',
self,
min=0,
step=1,
max=1000.,
label='Epsilon'))
self.menu.append(
ui.Slider('dist_threshold',
self,
min=1,
step=1,
max=20000,
label='Distance threshold'))
self.menu.append(
ui.Slider('delta_area_threshold',
self,
min=0,
step=0.1,
max=20,
label='Area threshold'))
self.menu.append(
ui.Slider('threshold',
self,
min=0,
step=1,
max=255,
label='Threshold'))
self.menu.append(
ui.Slider('expected_contours',
self,
min=1,
step=1,
max=32,
label='Expected contours (not working yet)'))
self.menu.append(ui.Switch('show_edges', self, label='Show edges'))
color_menu = ui.Growing_Menu('Colors')
color_menu.collapsed = True
color_menu.append(ui.Info_Text('Outside Color'))
color_menu.append(
ui.Slider('r', self, min=0.0, step=0.05, max=1.0, label='Red'))
color_menu.append(
ui.Slider('g', self, min=0.0, step=0.05, max=1.0, label='Green'))
color_menu.append(
ui.Slider('b', self, min=0.0, step=0.05, max=1.0, label='Blue'))
color_menu.append(
ui.Slider('a', self, min=0.0, step=0.05, max=1.0, label='Alpha'))
self.menu.append(color_menu)
def main():
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
hdpi_factor = glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(
window)[0]
w, h = w * hdpi_factor, h * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfwMakeContextCurrent(active_window)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_close(window):
glfwSetWindowShouldClose(main_window, True)
logger.debug('Process closing from window')
try:
rec_dir = sys.argv[1]
except:
#for dev, supply hardcoded dir:
rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
if os.path.isdir(rec_dir):
logger.debug("Dev option: Using hadcoded data dir.")
else:
if getattr(sys, 'frozen', False):
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease drag a Pupil recoding directory onto the launch icon."
)
else:
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease supply a Pupil recoding directory as first arg when calling Pupil Player."
)
return
if not is_pupil_rec_dir(rec_dir):
logger.error(
"You did not supply a dir with the required files inside.")
return
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
#backwards compatibility fn.
patch_meta_info(rec_dir)
#parse info.csv file
meta_info_path = rec_dir + "/info.csv"
with open(meta_info_path) as info:
meta_info = dict(
((line.strip().split('\t')) for line in info.readlines()))
rec_version = read_rec_version(meta_info)
if rec_version < VersionFormat('0.4'):
video_path = rec_dir + "/world.avi"
timestamps_path = rec_dir + "/timestamps.npy"
else:
video_path = rec_dir + "/world.mkv"
timestamps_path = rec_dir + "/world_timestamps.npy"
gaze_positions_path = rec_dir + "/gaze_positions.npy"
#load gaze information
gaze_list = np.load(gaze_positions_path)
timestamps = np.load(timestamps_path)
#correlate data
if rec_version < VersionFormat('0.4'):
positions_by_frame = correlate_gaze_legacy(gaze_list, timestamps)
else:
positions_by_frame = correlate_gaze(gaze_list, timestamps)
# Initialize capture
cap = autoCreateCapture(video_path, timestamps=timestamps_path)
if isinstance(cap, FakeCapture):
logger.error("could not start capture.")
return
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position',
(0, 0)) # not yet using this one.
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# create container for globally scoped vars (within world)
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.gaze_list = gaze_list
g_pool.positions_by_frame = positions_by_frame
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def get_scale():
return g_pool.gui.scale
def open_plugin(plugin):
if plugin == "Select to load":
return
logger.debug('Open Plugin: %s' % plugin)
new_plugin = plugin(g_pool)
g_pool.plugins.add(new_plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Scrolling_Menu("Settings",
pos=(-350, 20),
size=(300, 300))
g_pool.main_menu.append(ui.Button("quit", lambda: on_close(None)))
g_pool.main_menu.configuration = session_settings.get(
'main_menu_config', {})
g_pool.main_menu.append(
ui.Slider('scale',
setter=set_scale,
getter=get_scale,
step=.05,
min=0.75,
max=2.5,
label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
rec_version))
g_pool.main_menu.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label='Play',
hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend(
[g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Scan_Path', {}), ('Vis_Polyline', {}),
('Vis_Circle', {}), ('Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
glfwSetWindowPos(main_window, 0, 0)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture((height, width, 3))
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_now() - .03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.get_cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play = False
if g_pool.new_seek:
display_time = new_frame.timestamp
g_pool.new_seek = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#new positons we make a deepcopy just like the image is a copy.
events['pupil_positions'] = deepcopy(positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
update_named_texture(g_pool.image_tex, frame.img)
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time() - timestamp
sleep(wait_time - spent_time)
except:
pass
timestamp = time()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['main_menu_config'] = g_pool.main_menu.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def update_menu(self):
try:
del self.menu[:]
except AttributeError:
return
from pyglui import ui
if self.device is None:
self.menu.append(ui.Info_Text("Capture initialization failed."))
return
def align_and_restart(val):
self.align_streams = val
self.restart_device()
self.menu.append(
ui.Switch("record_depth", self, label="Record Depth Stream"))
self.menu.append(
ui.Switch("preview_depth", self, label="Preview Depth"))
self.menu.append(
ui.Switch("align_streams",
self,
label="Align Streams",
setter=align_and_restart))
def toggle_depth_display():
def on_depth_mouse_button(window, button, action, mods):
if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS:
self.mouse_drag = True
if (button == glfw.GLFW_MOUSE_BUTTON_LEFT
and action == glfw.GLFW_RELEASE):
self.mouse_drag = False
if self.depth_window is None:
self.pitch = 0
self.yaw = 0
win_size = glfw.glfwGetWindowSize(self.g_pool.main_window)
self.depth_window = glfw.glfwCreateWindow(
win_size[0], win_size[1], "3D Point Cloud")
glfw.glfwSetMouseButtonCallback(self.depth_window,
on_depth_mouse_button)
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self.depth_window)
gl_utils.basic_gl_setup()
gl_utils.make_coord_system_norm_based()
# refresh speed settings
glfw.glfwSwapInterval(0)
glfw.glfwMakeContextCurrent(active_window)
native_presets = [
("None", None),
("Best Quality", rs_preset.RS_PRESET_BEST_QUALITY),
("Largest image", rs_preset.RS_PRESET_LARGEST_IMAGE),
("Highest framerate", rs_preset.RS_PRESET_HIGHEST_FRAMERATE),
]
def set_stream_preset(val):
if self.stream_preset != val:
self.stream_preset = val
self.restart_device()
self.menu.append(
ui.Selector(
"stream_preset",
self,
setter=set_stream_preset,
labels=[preset[0] for preset in native_presets],
selection=[preset[1] for preset in native_presets],
label="Stream preset",
))
color_sizes = sorted(self._available_modes[rs_stream.RS_STREAM_COLOR],
reverse=True)
selector = ui.Selector(
"frame_size",
self,
# setter=,
selection=color_sizes,
label="Resolution" if self.align_streams else "Color Resolution",
)
selector.read_only = self.stream_preset is not None
self.menu.append(selector)
def color_fps_getter():
avail_fps = [
fps for fps in self._available_modes[rs_stream.RS_STREAM_COLOR]
[self.frame_size] if self.depth_frame_rate % fps == 0
]
return avail_fps, [str(fps) for fps in avail_fps]
selector = ui.Selector(
"frame_rate",
self,
# setter=,
selection_getter=color_fps_getter,
label="Color Frame Rate",
)
selector.read_only = self.stream_preset is not None
self.menu.append(selector)
if not self.align_streams:
depth_sizes = sorted(
self._available_modes[rs_stream.RS_STREAM_DEPTH], reverse=True)
selector = ui.Selector(
"depth_frame_size",
self,
# setter=,
selection=depth_sizes,
label="Depth Resolution",
)
selector.read_only = self.stream_preset is not None
self.menu.append(selector)
def depth_fps_getter():
avail_fps = [
fps for fps in self._available_modes[rs_stream.RS_STREAM_DEPTH]
[self.depth_frame_size] if fps % self.frame_rate == 0
]
return avail_fps, [str(fps) for fps in avail_fps]
selector = ui.Selector(
"depth_frame_rate",
self,
selection_getter=depth_fps_getter,
label="Depth Frame Rate",
)
selector.read_only = self.stream_preset is not None
self.menu.append(selector)
def reset_options():
if self.device:
try:
self.device.reset_device_options_to_default(
self.controls.keys())
except pyrs.RealsenseError as err:
logger.info("Resetting some device options failed")
logger.debug("Reason: {}".format(err))
finally:
self.controls.refresh()
self.menu.append(ui.Button("Point Cloud Window", toggle_depth_display))
sensor_control = ui.Growing_Menu(label="Sensor Settings")
sensor_control.append(
ui.Button("Reset device options to default", reset_options))
for ctrl in sorted(self.controls.values(),
key=lambda x: x.range.option):
# sensor_control.append(ui.Info_Text(ctrl.description))
if (ctrl.range.min == 0.0 and ctrl.range.max == 1.0
and ctrl.range.step == 1.0):
sensor_control.append(
ui.Switch("value",
ctrl,
label=ctrl.label,
off_val=0.0,
on_val=1.0))
else:
sensor_control.append(
ui.Slider(
"value",
ctrl,
label=ctrl.label,
min=ctrl.range.min,
max=ctrl.range.max,
step=ctrl.range.step,
))
self.menu.append(sensor_control)
def eye(g_pool, cap_src, cap_size, rx_from_world, eye_id=0):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,
'eye%s.log' % eye_id),
mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level + 10)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'Eye' + str(eye_id) +
' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'EYE' + str(eye_id) +
' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == GLFW_RELEASE and u_r.active_edit_pt:
u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos, (frame.width, frame.height)) # Position in img pixels
if u_r.mouse_over_edit_pt(pos, u_r.handle_size + 40,
u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if u_r.active_edit_pt:
pos = normalize((x, y), glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
u_r.move_vertex(u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
cap.frame_size = cap_size
cap.frame_rate = 90 #default
cap.settings = session_settings.get('capture_settings', {})
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.display_mode = session_settings.get('display_mode', 'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be a small as possible but big enough to capture to pupil in its movements",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters with in the Pupil Detection menu below."
}
# g_pool.draw_pupil = session_settings.get('draw_pupil',True)
u_r = UIRoi(frame.img.shape)
u_r.set(session_settings.get('roi', u_r.get()))
writer = None
pupil_detector = Canny_Detector(g_pool)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
# Initialize glfw
glfwInit()
if g_pool.binocular:
title = "Binocular eye %s" % eye_id
else:
title = 'Eye'
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl_init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex, frame.img)
glfwSwapInterval(0)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
g_pool.gui.append(
ui.Hot_Key("quit",
setter=on_close,
getter=lambda: True,
label="X",
hotkey=GLFW_KEY_ESCAPE))
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# let detector add its GUI
pupil_detector.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if rx_from_world.poll():
command, raw_mode = rx_from_world.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(record_path,
"eye%s_timestamps.npy" % eye_id)
if raw_mode and hasattr(frame, 'jpeg_buffer'):
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path, cap.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mkv" % eye_id)
writer = CV_Writer(video_path, float(cap.frame_rate),
cap.frame_size)
timestamps = []
else:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(
frame, user_roi=u_r, visualize=g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
update_named_texture(g_pool.image_tex, frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
update_named_texture(g_pool.image_tex, frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
draw_named_texture(g_pool.image_tex)
# switch to work in pixel space
make_coord_system_pixel_based((frame.height, frame.width, 3),
g_pool.flip)
if result['confidence'] > 0:
if result.has_key('axes'):
pts = cv2.ellipse2Poly(
(int(result['center'][0]), int(result['center'][1])),
(int(result['axes'][0] / 2), int(result['axes'][1] / 2)),
int(result['angle']), 0, 360, 15)
cygl_draw_polyline(pts, 1, cygl_rgba(1., 0, 0, .5))
cygl_draw_points([result['center']],
size=20,
color=cygl_rgba(1., 0., 0., .5),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
u_r.draw(g_pool.gui.scale)
#update screen
glfwSwapBuffers(main_window)
glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
pupil_detector.cleanup()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
#flushing queue in case world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def player(rec_dir, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, app_version):
# general imports
from time import sleep
import logging
from glob import glob
from time import time, strftime, localtime
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
# imports
from file_methods import Persistent_Dict, load_object, next_export_sub_dir
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture, RGBA
import gl_utils
# capture
from video_capture import init_playback_source, EndofVideoError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
from csv_utils import write_key_value_file
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_control import Seek_Control
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
# from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from batch_exporter import Batch_Exporter, Batch_Export
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
from system_graphs import System_Graphs
from system_timelines import System_Timelines
from blink_detection import Offline_Blink_Detection
assert VersionFormat(pyglui_version) >= VersionFormat('1.18'), 'pyglui out of date, please upgrade to newest version'
runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
system_plugins = [Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export, System_Timelines]
user_plugins = [Vis_Circle,
Vis_Fixation,
Vis_Polyline,
Vis_Light_Points,
Vis_Cross,
Vis_Watermark,
Vis_Eye_Video_Overlay,
Vis_Scan_Path,
Offline_Fixation_Detector,
Offline_Blink_Detection,
Batch_Exporter,
Video_Export_Launcher,
Offline_Surface_Tracker,
Raw_Data_Exporter,
Annotation_Player,
Log_History,
Pupil_From_Recording,
Offline_Pupil_Detection,
Gaze_From_Recording,
Offline_Calibration
] + runtime_plugins
plugins = system_plugins + user_plugins
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal hdpi_factor
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
g_pool.camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
icon_bar_width = 50
window_size = None
hdpi_factor = 1.0
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg', '.fake')
video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in valid_ext][0]
init_playback_source(g_pool, source_path=video_path)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version", '0.0')) != app_version:
logger.info("Session setting are a different version of this app. I will not use those.")
session_settings.clear()
g_pool.capture.playback_speed = session_settings.get('playback_speed', 1.)
width, height = g_pool.capture.frame_size
width += icon_bar_width
width, height = session_settings.get('window_size', (width, height))
window_pos = session_settings.get('window_position', window_position_default)
window_name = "Pupil Player: {} - {}".format(meta_info["Recording Name"],
os.path.split(rec_dir)[-1])
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, window_name, None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = (g_pool.camera_render_size[0] + int(icon_bar_width*g_pool.gui_user_scale*hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1])
logger.warning(icon_bar_width*g_pool.gui_user_scale*hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.min_calibration_confidence = session_settings.get('min_calibration_confidence', 0.8)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
left_idx = g_pool.seek_control.trim_left
right_idx = g_pool.seek_control.trim_right
export_range = left_idx, right_idx + 1 # exclusive range.stop
export_dir = os.path.join(g_pool.rec_dir, 'exports')
export_dir = next_export_sub_dir(export_dir)
os.makedirs(export_dir)
logger.info('Created export dir at "{}"'.format(export_dir))
export_info = {'Player Software Version': str(g_pool.version),
'Data Format Version': meta_info['Data Format Version'],
'Export Date': strftime("%d.%m.%Y", localtime()),
'Export Time': strftime("%H:%M:%S", localtime()),
'Frame Index Range:': g_pool.seek_control.get_frame_index_trim_range_string(),
'Relative Time Range': g_pool.seek_control.get_rel_time_trim_range_string(),
'Absolute Time Range': g_pool.seek_control.get_abs_time_trim_range_string()}
with open(os.path.join(export_dir, 'export_info.csv'), 'w') as csv:
write_key_value_file(csv, export_info)
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
ipc_pub.notify({'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2.})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left')
g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden')
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.),
size=(0., 0.), header_pos='headline')
g_pool.user_timelines.color = RGBA(a=0.)
g_pool.user_timelines.collapsed = True
# add container that constaints itself to the seekbar height
vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
vert_constr.append(g_pool.user_timelines)
g_pool.timelines.append(vert_constr)
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
general_settings = ui.Growing_Menu('General', header_pos='headline')
general_settings.append(ui.Button('Reset window size', set_window_size))
general_settings.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2]+list(np.arange(1.5, 5.1, .5)), label='Interface Size'))
general_settings.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
general_settings.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
general_settings.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
general_settings.append(ui.Info_Text('High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence.'))
general_settings.append(ui.Slider('min_data_confidence', g_pool,
setter=set_data_confidence,
step=.05, min=0.0, max=1.0,
label='Minimum data confidence'))
general_settings.append(ui.Button('Restart with default settings', reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons')
icon.tooltip = 'General Settings'
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100))
g_pool.export_button = ui.Thumb('export',
label=chr(0xe2c5),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='pupil_icons')
g_pool.quickbar.extend([g_pool.export_button])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [('Plugin_Manager', {}),
('Seek_Control', {}),
('Log_Display', {}),
('Raw_Data_Exporter', {}),
('Vis_Polyline', {}),
('Vis_Circle', {}),
('System_Graphs', {}),
('System_Timelines', {}),
('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}),
('Gaze_From_Recording', {})]
g_pool.plugins = Plugin_List(g_pool, session_settings.get('loaded_plugins', default_plugins))
general_settings.insert(-1 ,ui.Text_Input('rel_time_trim_section',
getter=g_pool.seek_control.get_rel_time_trim_range_string,
setter=g_pool.seek_control.set_rel_time_trim_range_string,
label='Relative time range to export'))
general_settings.insert(-1 ,ui.Text_Input('frame_idx_trim_section',
getter=g_pool.seek_control.get_frame_index_trim_range_string,
setter=g_pool.seek_control.set_frame_index_trim_range_string,
label='Frame index range to export'))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
toggle_general_settings(True)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(
g_pool.plugin_by_name[n['name']], args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# grab new frame
if g_pool.capture.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = g_pool.capture.get_frame()
except EndofVideoError:
# end of video logic: pause at last frame.
g_pool.capture.play = False
logger.warning("End of video")
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# pupil and gaze positions are added by their respective producer plugins
events['pupil_positions'] = []
events['gaze_positions'] = []
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture._recent_frame = frame
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ''
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode())
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in user_input.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in user_input.chars:
for p in g_pool.plugins:
p.on_char(char_)
glfw.glfwSwapBuffers(main_window)
# present frames at appropriate speed
g_pool.capture.wait(frame)
glfw.glfwPollEvents()
session_settings['playback_speed'] = g_pool.capture.playback_speed
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['min_calibration_confidence'] = g_pool.min_calibration_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings['window_size'] = session_window_size
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.capture.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except:
import traceback
trace = traceback.format_exc()
logger.error('Process Player crashed with trace:\n{}'.format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
sleep(1.0)
def update_gui_markers(self):
def close():
self.alive = False
def set_min_marker_perimeter(val):
self.min_marker_perimeter = val
self.notify_all({
'subject': 'min_marker_perimeter_changed',
'delay': 1
})
def set_invert_image(val):
self.invert_image = val
self.invalidate_marker_cache()
self.invalidate_surface_caches()
self.menu.elements[:] = []
self.menu.append(ui.Button('Close', close))
self.menu.append(
ui.Switch('invert_image',
self,
setter=set_invert_image,
label='Use inverted markers'))
self.menu.append(
ui.Slider('min_marker_perimeter',
self,
min=20,
max=500,
step=1,
setter=set_min_marker_perimeter))
self.menu.append(
ui.Info_Text(
'The offline surface tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.'
))
self.menu.append(
ui.Info_Text(
"Press the export button or type 'e' to start the export."))
self.menu.append(
ui.Selector('mode',
self,
label='Mode',
selection=[
"Show Markers and Surfaces", "Show marker IDs",
"Show Heatmaps", "Show Metrics"
]))
self.menu.append(
ui.Info_Text(
'To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.'
))
self.menu.append(
ui.Button("(Re)-calculate gaze distributions", self.recalculate))
self.menu.append(ui.Button("Add surface", lambda: self.add_surface()))
for s in self.surfaces:
idx = self.surfaces.index(s)
s_menu = ui.Growing_Menu("Surface {}".format(idx))
s_menu.collapsed = True
s_menu.append(ui.Text_Input('name', s))
s_menu.append(ui.Text_Input('x', s.real_world_size,
label='X size'))
s_menu.append(ui.Text_Input('y', s.real_world_size,
label='Y size'))
s_menu.append(ui.Button('Open Debug Window', s.open_close_window))
#closure to encapsulate idx
def make_remove_s(i):
return lambda: self.remove_surface(i)
remove_s = make_remove_s(idx)
s_menu.append(ui.Button('remove', remove_s))
self.menu.append(s_menu)
def world(g_pool, cap_src, cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from eye process[es]
Can run various plug-ins.
"""
#manage plugins
runtime_plugins = import_runtime_plugins(
os.path.join(g_pool.user_dir, 'plugins'))
user_launchable_plugins = [
Show_Calibration, Pupil_Server, Pupil_Sync, Marker_Detector
] + runtime_plugins
system_plugins = [Log_Display, Display_Recent_Gaze, Recorder]
plugin_by_index = system_plugins + user_launchable_plugins + calibration_plugins + gaze_mapping_plugins
name_by_index = [p.__name__ for p in plugin_by_index]
plugin_by_name = dict(zip(name_by_index, plugin_by_index))
default_plugins = [('Log_Display', {}), ('Dummy_Gaze_Mapper', {}),
('Display_Recent_Gaze', {}),
('Screen_Marker_Calibration', {}), ('Recorder', {})]
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconfied):
pass
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
tick = delta_t()
def get_dt():
return next(tick)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
if session_settings.get("version", VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
cap.frame_size = cap_size
cap.frame_rate = 24 #default
cap.settings = session_settings.get('capture_settings', {})
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.update_textures = session_settings.get("update_textures", 2)
g_pool.capture = cap
g_pool.pupil_confidence_threshold = session_settings.get(
'pupil_confidence_threshold', .6)
g_pool.active_calibration_plugin = None
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
#window and gl setup
glfwInit()
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfwCreateWindow(width, height, "World")
window_pos = session_settings.get('window_position',
window_position_default)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-350, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface size'))
general_settings.append(
ui.Button(
'Reset window size',
lambda: glfwSetWindowSize(main_window, frame.width, frame.height)))
general_settings.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.sidebar.append(general_settings)
advanced_settings = ui.Growing_Menu('Advanced')
advanced_settings.append(
ui.Selector('update_textures',
g_pool,
label="Update display",
selection=range(3),
labels=('No update', 'Gray', 'Color')))
advanced_settings.append(
ui.Slider('pupil_confidence_threshold',
g_pool,
step=.01,
min=0.,
max=1.,
label='Minimum pupil confidence'))
advanced_settings.append(
ui.Info_Text('Capture Version: %s' % g_pool.version))
general_settings.append(advanced_settings)
g_pool.calibration_menu = ui.Growing_Menu('Calibration')
g_pool.calibration_menu.append(
ui.Selector(
'active_calibration_plugin',
getter=lambda: g_pool.active_calibration_plugin.__class__,
selection=calibration_plugins,
labels=[p.__name__.replace('_', ' ') for p in calibration_plugins],
setter=open_plugin,
label='Method'))
g_pool.sidebar.append(g_pool.calibration_menu)
g_pool.gui.append(g_pool.sidebar)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(
ui.Hot_Key("quit",
setter=on_close,
getter=lambda: True,
label="X",
hotkey=GLFW_KEY_ESCAPE))
g_pool.capture.init_gui(g_pool.sidebar)
#plugins that are loaded based on user settings from previous session
g_pool.notifications = []
g_pool.plugins = Plugin_List(
g_pool, plugin_by_name,
session_settings.get('loaded_plugins', default_plugins))
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetWindowIconifyCallback(main_window, on_iconify)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img.shape)
update_named_texture(g_pool.image_tex, frame.img)
# refresh speed settings
glfwSwapInterval(0)
#trigger setup of window and gl sizes
on_resize(main_window, *glfwGetFramebufferSize(main_window))
#now the we have aproper window we can load the last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 130)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from camera failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video file is done. Stopping")
break
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
#a dictionary that allows plugins to post and read events
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
recent_pupil_positions.append(p)
pupil_graph.add(p['confidence'])
events['pupil_positions'] = recent_pupil_positions
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
if g_pool.update_textures == 2:
update_named_texture(g_pool.image_tex, frame.img)
elif g_pool.update_textures == 1:
update_named_texture(g_pool.image_tex, frame.gray)
make_coord_system_norm_based()
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based((frame.height, frame.width, 3))
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings[
'pupil_confidence_threshold'] = g_pool.pupil_confidence_threshold
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['update_textures'] = g_pool.update_textures
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
glfwTerminate()
cap.close()
logger.debug("Process done")
def demo():
global quit
quit = False
# Callback functions
def on_resize(window,w, h):
h = max(h,1)
w = max(w,1)
hdpi_factor = glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0]
w,h = w*hdpi_factor,h*hdpi_factor
gui.update_window(w,h)
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
# norm_size = normalize((w,h),glfwGetWindowSize(window))
# fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
adjust_gl_view(w,h,window)
glfwMakeContextCurrent(active_window)
def on_iconify(window,iconfied):
pass
def on_key(window, key, scancode, action, mods):
gui.update_key(key,scancode,action,mods)
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
if mods == GLFW_MOD_SUPER:
if key == 67:
# copy value to system clipboard
# ideally copy what is in our text input area
test_val = "copied text input"
glfwSetClipboardString(window,test_val)
print("set clipboard to: %s" %(test_val))
if key == 86:
# copy from system clipboard
clipboard = glfwGetClipboardString(window)
print("pasting from clipboard: %s" %(clipboard))
def on_char(window,char):
gui.update_char(char)
def on_button(window,button, action, mods):
gui.update_button(button,action,mods)
# pos = normalize(pos,glfwGetWindowSize(window))
# pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
gui.update_mouse(x,y)
def on_scroll(window,x,y):
gui.update_scroll(x,y)
def on_close(window):
global quit
quit = True
logger.info('Process closing from window')
# get glfw started
glfwInit()
window = glfwCreateWindow(width, height, "pyglui demo", None, None)
if not window:
exit()
glfwSetWindowPos(window,0,0)
# Register callbacks for the window
glfwSetWindowSizeCallback(window,on_resize)
glfwSetWindowCloseCallback(window,on_close)
glfwSetWindowIconifyCallback(window,on_iconify)
glfwSetKeyCallback(window,on_key)
glfwSetCharCallback(window,on_char)
glfwSetMouseButtonCallback(window,on_button)
glfwSetCursorPosCallback(window,on_pos)
glfwSetScrollCallback(window,on_scroll)
# test out new paste function
glfwMakeContextCurrent(window)
init()
basic_gl_setup()
print(glGetString(GL_VERSION))
class Temp(object):
"""Temp class to make objects"""
def __init__(self):
pass
foo = Temp()
foo.bar = 34
foo.sel = 'mi'
foo.selection = ['€','mi', u"re"]
foo.mytext = "some text"
def set_text_val(val):
foo.mytext = val
# print 'setting to :',val
print("pyglui version: %s" %(ui.__version__))
gui = ui.UI()
gui.scale = 1.0
sidebar = ui.Scrolling_Menu("MySideBar",pos=(-300,0),size=(0,0),header_pos='left')
sm = ui.Growing_Menu("SubMenu",pos=(0,0),size=(0,100))
sm.append(ui.Slider("bar",foo))
sm.append(ui.Text_Input('mytext',foo,setter=set_text_val))
sm.append(ui.Selector('sel',foo,selection=foo.selection))
sidebar.append(sm)
gui.append(sidebar)
menu = ui.Scrolling_Menu('My Window 1', pos=(200,100), size=(50, 200),header_pos='top')
menu.append(ui.Slider("bar",foo))
gui.append(menu)
menu = ui.Scrolling_Menu('My Window 2', pos=(200,100), size=(50, 200),header_pos='top')
menu.append(ui.Text_Input('mytext',foo,setter=set_text_val))
gui.append(menu)
label = 'Ï'
# label = 'R'
gui.append(ui.Thumb('mytext',foo,label=label,hotkey='r',label_font='fontawesome',label_offset_x=5,label_offset_y=0,label_offset_size=-20))
import os
import psutil
pid = os.getpid()
ps = psutil.Process(pid)
ts = time.time()
from pyglui import graph
print(graph.__version__)
cpu_g = graph.Line_Graph()
cpu_g.pos = (50,100)
cpu_g.update_fn = ps.cpu_percent
cpu_g.update_rate = 5
cpu_g.label = 'CPU %0.1f'
fps_g = graph.Line_Graph()
fps_g.pos = (50,100)
fps_g.update_rate = 5
fps_g.label = "%0.0f FPS"
fps_g.color[:] = .1,.1,.8,.9
on_resize(window,*glfwGetWindowSize(window))
while not quit:
dt,ts = time.time()-ts,time.time()
clear_gl_screen()
draw_concentric_circles( (500,250), 200, 6 , 1.0 )
draw_concentric_circles( (600,250), 200, 7 , 1.0 )
draw_concentric_circles( (700,250), 200, 8 , 0.1 )
cpu_g.update()
cpu_g.draw()
fps_g.add(1./dt)
fps_g.draw()
gui.update()
glfwSwapBuffers(window)
glfwPollEvents()
gui.terminate()
glfwTerminate()
logger.debug("Process done")
def recent_events(self, events):
frame = events.get("frame")
if not frame:
return
self.current_frame_idx = frame.index
if self.video_export_queue:
self.video_export(self.video_export_queue.pop(0))
if self.surface_export_queue:
self.surface_export(self.surface_export_queue.pop(0))
if self.sections == None:
plugins = [
p for p in self.g_pool.plugins if isinstance(p, Offline_Surface_Tracker)
]
if plugins:
marker_tracker_plugin = plugins[0]
else:
self.update_bar_indicator(False)
return
if marker_tracker_plugin.cache.complete:
# make a marker signal 0 = none, 1 = in, -1=out
in_id = self.in_marker_id
out_id = self.out_marker_id
logger.debug(
"Looking for trim mark markers: {},{}".format(in_id, out_id)
)
in_out_signal = [0] * len(marker_tracker_plugin.cache)
for idx, frame in enumerate(marker_tracker_plugin.cache):
# marker = {'id':msg,'verts':r,'verts_norm':r_norm,'centroid':centroid,"frames_since_true_detection":0}
for marker in frame:
if marker["id"] == in_id:
in_out_signal[idx] += 1
if marker["id"] == out_id:
in_out_signal[idx] -= 1
# make a smooth signal
in_out_smooth = np.convolve(
in_out_signal, [2.0 / 30] * 30, mode="same"
) # mean filter with sum 2 and len 60,
# Mode 'same' returns output of length max(signal, filter).
# find falling edges of in markers clusters
i = -1
in_falling_edge_idx = []
for t, g in groupby(in_out_smooth, lambda x: x >= 1):
s = i + 1
i += len(list(g))
if t:
in_falling_edge_idx.append(i)
# find rising edges of out markers clusters
i = -1
out_rising_edge_idx = []
for t, g in groupby(in_out_smooth, lambda x: x <= -1):
s = i + 1
i += len(list(g))
if t:
out_rising_edge_idx.append(s)
events = [("out", idx) for idx in out_rising_edge_idx] + [
("in", idx) for idx in in_falling_edge_idx
]
manual_markers = [("in", idx) for idx in self.man_in_marks] + [
("out", idx) for idx in self.man_out_marks
]
events += manual_markers
events.sort(key=lambda x: x[1])
events = (
[("in", 0)] + events + [("out", len(marker_tracker_plugin.cache))]
)
self.sections = []
for t, g in groupby(events, lambda x: x[0]):
if t == "in":
last_in_marker_of_this_cluster = list(g)[-1]
section_in_index = last_in_marker_of_this_cluster[1]
else:
# t=="out"
fist_out_marker_of_this_clutser = (
g.next()
) # first item in cluster
section_out_index = fist_out_marker_of_this_clutser[1]
self.sections.append((section_in_index, section_out_index))
self.sections = [
(s, e) for s, e in self.sections if e - s > 10
] # we filter out tiny sections
# because they can happen with out markers at video start and in marker at video end.
# Lines for areas that have been cached
self.gl_display_ranges = []
for r in self.sections: # [[0,1],[3,4]]
self.gl_display_ranges += (
(r[0], 0),
(r[1], 0),
) # [(0,0),(1,0),(3,0),(4,0)]
if self.sections:
self.activate_section = self.sections[0]
del self.menu.elements[:]
self.menu.append(
ui.Slider(
"in_marker_id",
self,
min=0,
step=1,
max=63,
label="IN marker id",
)
)
self.menu.append(
ui.Slider(
"out_marker_id",
self,
min=0,
step=1,
max=63,
label="OUT marker id",
)
)
self.menu.append(
ui.Selector(
"active_section",
self,
selection=self.sections,
setter=self.activate_section,
label="set section",
)
)
self.menu.append(
ui.Button("video export all sections", self.enqueue_video_export)
)
self.menu.append(
ui.Button(
"surface export all sections", self.enqueue_surface_export
)
)
self.menu.append(ui.Button("add in_mark here", self.add_manual_in_mark))
self.menu.append(
ui.Selector(
"man_in_mark",
selection=self.man_in_marks,
setter=self.del_man_in_mark,
getter=lambda: "select one",
label="del manual in marker",
)
)
self.menu.append(
ui.Button("add out mark here", self.add_manual_out_mark)
)
self.menu.append(
ui.Selector(
"man_out_mark",
selection=self.man_out_marks,
setter=self.del_man_out_mark,
getter=lambda: "select one",
label="del manual out marker",
)
)
else:
self.menu.label = "Marker Auto Trim Marks: Waiting for Cacher to finish"
def init_ui(self):
self.add_menu()
self.menu_icon.order = 0.01
self.menu.label = "System Graphs"
self.menu.append(ui.Switch("show_cpu", self, label="Display CPU usage"))
self.menu.append(ui.Switch("show_fps", self, label="Display frames per second"))
self.menu.append(
ui.Switch("show_conf0", self, label="Display confidence for eye 0")
)
self.menu.append(
ui.Switch("show_conf1", self, label="Display confidence for eye 1")
)
self.menu.append(
ui.Switch("show_dia0", self, label="Display pupil diameter for eye 0")
)
self.menu.append(
ui.Switch("show_dia1", self, label="Display pupil diameter for eye 1")
)
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
self.cpu_graph = graph.Bar_Graph()
self.cpu_graph.pos = (20, 50)
self.cpu_graph.update_fn = ps.cpu_percent
self.cpu_graph.update_rate = 5
self.cpu_graph.label = "CPU %0.1f"
self.fps_graph = graph.Bar_Graph()
self.fps_graph.pos = (140, 50)
self.fps_graph.update_rate = 5
self.fps_graph.label = "%0.0f FPS"
self.conf0_graph = graph.Bar_Graph(max_val=1.0)
self.conf0_graph.pos = (260, 50)
self.conf0_graph.update_rate = 5
self.conf0_graph.label = "id0 conf: %0.2f"
self.conf1_graph = graph.Bar_Graph(max_val=1.0)
self.conf1_graph.pos = (380, 50)
self.conf1_graph.update_rate = 5
self.conf1_graph.label = "id1 conf: %0.2f"
self.dia0_graph = graph.Bar_Graph(min_val=self.dia_min, max_val=self.dia_max)
self.dia0_graph.pos = (260, 100)
self.dia0_graph.update_rate = 5
self.dia0_graph.label = "id0 dia: %0.2f"
self.dia1_graph = graph.Bar_Graph(min_val=self.dia_min, max_val=self.dia_max)
self.dia1_graph.pos = (380, 100)
self.dia1_graph.update_rate = 5
self.dia1_graph.label = "id1 dia: %0.2f"
self.conf_grad = (
RGBA(1.0, 0.0, 0.0, self.conf0_graph.color[3]),
self.conf0_graph.color,
)
def set_dia_min(val):
self.dia0_graph.min_val = val
self.dia1_graph.min_val = val
def set_dia_max(val):
self.dia0_graph.max_val = val
self.dia1_graph.max_val = val
self.menu.append(
ui.Slider(
"min_val",
self.dia0_graph,
label="Minimum pupil diameter",
setter=set_dia_min,
min=0.0,
max=15.0,
step=0.1,
)
)
self.menu.append(
ui.Slider(
"max_val",
self.dia0_graph,
label="Maximum pupil diameter",
setter=set_dia_max,
min=1.0,
max=15.0,
step=0.1,
)
)
self.on_window_resize(self.g_pool.main_window)
def update_menu(self):
try:
del self.menu[:]
except AttributeError:
return
from pyglui import ui
if self.device is None:
self.menu.append(ui.Info_Text('Capture initialization failed.'))
return
def align_and_restart(val):
self.align_streams = val
self.restart_device()
self.menu.append(
ui.Switch('record_depth', self, label='Record Depth Stream'))
self.menu.append(
ui.Switch('preview_depth', self, label='Preview Depth'))
self.menu.append(
ui.Switch('align_streams',
self,
label='Align Streams',
setter=align_and_restart))
color_sizes = sorted(self._available_modes[rs_stream.RS_STREAM_COLOR],
reverse=True)
self.menu.append(
ui.Selector(
'frame_size',
self,
# setter=,
selection=color_sizes,
label='Resolution'
if self.align_streams else 'Color Resolution'))
def color_fps_getter():
avail_fps = self._available_modes[rs_stream.RS_STREAM_COLOR][
self.frame_size]
return avail_fps, [str(fps) for fps in avail_fps]
self.menu.append(
ui.Selector(
'frame_rate',
self,
# setter=,
selection_getter=color_fps_getter,
label='Color Frame Rate'))
if not self.align_streams:
depth_sizes = sorted(
self._available_modes[rs_stream.RS_STREAM_DEPTH], reverse=True)
self.menu.append(
ui.Selector(
'depth_frame_size',
self,
# setter=,
selection=depth_sizes,
label='Depth Resolution'))
def depth_fps_getter():
avail_fps = self._available_modes[rs_stream.RS_STREAM_DEPTH][
self.depth_frame_size]
return avail_fps, [str(fps) for fps in avail_fps]
self.menu.append(
ui.Selector('depth_frame_rate',
self,
selection_getter=depth_fps_getter,
label='Depth Frame Rate'))
def reset_options():
if self.device:
try:
self.device.reset_device_options_to_default(
self.controls.keys())
except pyrs.RealsenseError as err:
logger.info('Resetting some device options failed')
logger.debug('Reason: {}'.format(err))
finally:
self.controls.refresh()
sensor_control = ui.Growing_Menu(label='Sensor Settings')
sensor_control.append(
ui.Button('Reset device options to default', reset_options))
for ctrl in sorted(self.controls.values(),
key=lambda x: x.range.option):
# sensor_control.append(ui.Info_Text(ctrl.description))
if ctrl.range.min == 0.0 and ctrl.range.max == 1.0 and ctrl.range.step == 1.0:
sensor_control.append(
ui.Switch('value',
ctrl,
label=ctrl.label,
off_val=0.0,
on_val=1.0))
else:
sensor_control.append(
ui.Slider('value',
ctrl,
label=ctrl.label,
min=ctrl.range.min,
max=ctrl.range.max,
step=ctrl.range.step))
self.menu.append(sensor_control)
def init_gui(self, sidebar):
#lets define some helper functions:
def gui_load_defaults():
for c in self.capture.controls:
try:
c.value = c.def_val
except:
pass
def set_size(new_size):
self.frame_size = new_size
menu_conf = self.menu.configuration
self.deinit_gui()
self.init_gui(self.sidebar)
self.menu.configuration = menu_conf
def gui_update_from_device():
for c in self.capture.controls:
c.refresh()
def gui_init_cam_by_uid(requested_id):
if requested_id is None:
self.re_init_capture(None)
else:
for cam in uvc.device_list():
if cam['uid'] == requested_id:
if is_accessible(requested_id):
self.re_init_capture(requested_id)
else:
logger.error(
"The selected Camera is already in use or blocked."
)
return
logger.warning(
"could not reinit capture, src_id not valid anymore")
return
#create the menu entry
self.menu = ui.Growing_Menu(label='Camera Settings')
cameras = uvc.device_list()
camera_names = ['Fake Capture'] + [c['name'] for c in cameras]
camera_ids = [None] + [c['uid'] for c in cameras]
self.menu.append(
ui.Selector('uid',
self,
selection=camera_ids,
labels=camera_names,
label='Capture Device',
setter=gui_init_cam_by_uid))
sensor_control = ui.Growing_Menu(label='Sensor Settings')
sensor_control.append(
ui.Info_Text(
"Do not change these during calibration or recording!"))
sensor_control.collapsed = False
image_processing = ui.Growing_Menu(label='Image Post Processing')
image_processing.collapsed = True
sensor_control.append(
ui.Selector('frame_size',
self,
setter=set_size,
selection=self.capture.frame_sizes,
label='Resolution'))
sensor_control.append(
ui.Selector('frame_rate',
self,
selection=self.capture.frame_rates,
label='Framerate'))
for control in self.capture.controls:
c = None
ctl_name = control.display_name
#now we add controls
if control.d_type == bool:
c = ui.Switch('value',
control,
label=ctl_name,
on_val=control.max_val,
off_val=control.min_val)
elif control.d_type == int:
c = ui.Slider('value',
control,
label=ctl_name,
min=control.min_val,
max=control.max_val,
step=control.step)
elif type(control.d_type) == dict:
selection = [
value for name, value in control.d_type.iteritems()
]
labels = [name for name, value in control.d_type.iteritems()]
c = ui.Selector('value',
control,
label=ctl_name,
selection=selection,
labels=labels)
else:
pass
# if control['disabled']:
# c.read_only = True
# if ctl_name == 'Exposure, Auto Priority':
# # the controll should always be off. we set it to 0 on init (see above)
# c.read_only = True
if c is not None:
if control.unit == 'processing_unit':
image_processing.append(c)
else:
sensor_control.append(c)
self.menu.append(sensor_control)
if image_processing.elements:
self.menu.append(image_processing)
self.menu.append(ui.Button("refresh", gui_update_from_device))
self.menu.append(ui.Button("load defaults", gui_load_defaults))
self.sidebar = sidebar
#add below geneal settings
self.sidebar.insert(1, self.menu)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version):
# general imports
from time import sleep
import logging
from glob import glob
from time import time, strftime, localtime
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
from tasklib.background.patches import IPCLoggingPatch
IPCLoggingPatch.ipc_push_url = ipc_push_url
# imports
from file_methods import Persistent_Dict, next_export_sub_dir
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture, RGBA
import gl_utils
# capture
from video_capture import File_Source
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
import player_methods as pm
from pupil_recording import PupilRecording
from csv_utils import write_key_value_file
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from seek_control import Seek_Control
from surface_tracker import Surface_Tracker_Offline
# from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producer.gaze_from_recording import GazeFromRecording
from gaze_producer.gaze_from_offline_calibration import (
GazeFromOfflineCalibration,
)
from system_graphs import System_Graphs
from system_timelines import System_Timelines
from blink_detection import Offline_Blink_Detection
from audio_playback import Audio_Playback
from video_export.plugins.imotions_exporter import iMotions_Exporter
from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
from video_export.plugins.world_video_exporter import World_Video_Exporter
from head_pose_tracker.offline_head_pose_tracker import (
Offline_Head_Pose_Tracker,
)
from video_capture import File_Source
from video_overlay.plugins import Video_Overlay, Eye_Overlay
from pupil_recording import (
assert_valid_recording_type,
InvalidRecordingException,
)
assert VersionFormat(pyglui_version) >= VersionFormat(
"1.27"
), "pyglui out of date, please upgrade to newest version"
process_was_interrupted = False
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
nonlocal process_was_interrupted
process_was_interrupted = True
signal.signal(signal.SIGINT, interrupt_handler)
runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins"))
system_plugins = [
Log_Display,
Seek_Control,
Plugin_Manager,
System_Graphs,
System_Timelines,
Audio_Playback,
]
user_plugins = [
Vis_Circle,
Vis_Fixation,
Vis_Polyline,
Vis_Light_Points,
Vis_Cross,
Vis_Watermark,
Eye_Overlay,
Video_Overlay,
Offline_Fixation_Detector,
Offline_Blink_Detection,
Surface_Tracker_Offline,
Raw_Data_Exporter,
Annotation_Player,
Log_History,
Pupil_From_Recording,
Offline_Pupil_Detection,
GazeFromRecording,
GazeFromOfflineCalibration,
World_Video_Exporter,
iMotions_Exporter,
Eye_Video_Exporter,
Offline_Head_Pose_Tracker,
] + runtime_plugins
plugins = system_plugins + user_plugins
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal hdpi_factor
if w == 0 or h == 0:
return
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
for path in paths:
try:
assert_valid_recording_type(path)
_restart_with_recording(path)
return
except InvalidRecordingException as err:
logger.debug(str(err))
for plugin in g_pool.plugins:
if plugin.on_drop(paths):
break
def _restart_with_recording(rec_dir):
logger.debug("Starting new session with '{}'".format(rec_dir))
ipc_pub.notify(
{"subject": "player_drop_process.should_start", "rec_dir": rec_dir}
)
glfw.glfwSetWindowShouldClose(g_pool.main_window, True)
tick = delta_t()
def get_dt():
return next(tick)
recording = PupilRecording(rec_dir)
meta_info = recording.meta_info
# log info about Pupil Platform and Platform in player.log
logger.info("Application Version: {}".format(app_version))
logger.info("System Info: {}".format(get_system_info()))
icon_bar_width = 50
window_size = None
hdpi_factor = 1.0
# create container for globally scoped vars
g_pool = SimpleNamespace()
g_pool.app = "player"
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
video_path = recording.files().core().world().videos()[0].resolve()
File_Source(
g_pool,
timing="external",
source_path=video_path,
buffered_decoding=True,
fill_gaps=True,
)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player")
)
if VersionFormat(session_settings.get("version", "0.0")) != app_version:
logger.info(
"Session setting are a different version of this app. I will not use those."
)
session_settings.clear()
width, height = g_pool.capture.frame_size
width += icon_bar_width
width, height = session_settings.get("window_size", (width, height))
window_pos = session_settings.get("window_position", window_position_default)
window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}"
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, window_name, None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = (
g_pool.camera_render_size[0]
+ int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1],
)
logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.0
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get(
"min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT
)
g_pool.min_calibration_confidence = session_settings.get(
"min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT
)
# populated by producers
g_pool.pupil_positions = pm.Bisector()
g_pool.pupil_positions_by_id = (pm.Bisector(), pm.Bisector())
g_pool.gaze_positions = pm.Bisector()
g_pool.fixations = pm.Affiliator()
g_pool.eye_movements = pm.Affiliator()
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {"subject": "min_data_confidence_changed"}
notification["_notify_time_"] = time() + 0.8
g_pool.ipc_pub.notify(notification)
def do_export(_):
left_idx = g_pool.seek_control.trim_left
right_idx = g_pool.seek_control.trim_right
export_range = left_idx, right_idx + 1 # exclusive range.stop
export_ts_window = pm.exact_window(g_pool.timestamps, (left_idx, right_idx))
export_dir = os.path.join(g_pool.rec_dir, "exports")
export_dir = next_export_sub_dir(export_dir)
os.makedirs(export_dir)
logger.info('Created export dir at "{}"'.format(export_dir))
export_info = {
"Player Software Version": str(g_pool.version),
"Data Format Version": meta_info.min_player_version,
"Export Date": strftime("%d.%m.%Y", localtime()),
"Export Time": strftime("%H:%M:%S", localtime()),
"Frame Index Range:": g_pool.seek_control.get_frame_index_trim_range_string(),
"Relative Time Range": g_pool.seek_control.get_rel_time_trim_range_string(),
"Absolute Time Range": g_pool.seek_control.get_abs_time_trim_range_string(),
}
with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
write_key_value_file(csv, export_info)
notification = {
"subject": "should_export",
"range": export_range,
"ts_window": export_ts_window,
"export_dir": export_dir,
}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({"subject": "clear_settings_process.should_start"})
ipc_pub.notify(
{
"subject": "player_process.should_start",
"rec_dir": rec_dir,
"delay": 2.0,
}
)
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu(
"Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
)
g_pool.iconbar = ui.Scrolling_Menu(
"Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
)
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
g_pool.user_timelines = ui.Timeline_Menu(
"User Timelines", pos=(0.0, -150.0), size=(0.0, 0.0), header_pos="headline"
)
g_pool.user_timelines.color = RGBA(a=0.0)
g_pool.user_timelines.collapsed = True
# add container that constaints itself to the seekbar height
vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
vert_constr.append(g_pool.user_timelines)
g_pool.timelines.append(vert_constr)
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(ui.Button("Reset window size", set_window_size))
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)),
label="Interface Size",
)
)
general_settings.append(
ui.Info_Text(f"Minimum Player Version: {meta_info.min_player_version}")
)
general_settings.append(ui.Info_Text(f"Player Version: {g_pool.version}"))
general_settings.append(
ui.Info_Text(f"Recording Software: {meta_info.recording_software_name}")
)
general_settings.append(
ui.Info_Text(
f"Recording Software Version: {meta_info.recording_software_version}"
)
)
general_settings.append(
ui.Info_Text(
"High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
)
)
general_settings.append(
ui.Slider(
"min_data_confidence",
g_pool,
setter=set_data_confidence,
step=0.05,
min=0.0,
max=1.0,
label="Minimum data confidence",
)
)
general_settings.append(
ui.Button("Restart with default settings", reset_restart)
)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100), (100, -100))
g_pool.export_button = ui.Thumb(
"export",
label=chr(0xE2C5),
getter=lambda: False,
setter=do_export,
hotkey="e",
label_font="pupil_icons",
)
g_pool.quickbar.extend([g_pool.export_button])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [
("Plugin_Manager", {}),
("Seek_Control", {}),
("Log_Display", {}),
("Raw_Data_Exporter", {}),
("Vis_Polyline", {}),
("Vis_Circle", {}),
("System_Graphs", {}),
("System_Timelines", {}),
("World_Video_Exporter", {}),
("Pupil_From_Recording", {}),
("GazeFromRecording", {}),
("Audio_Playback", {}),
]
g_pool.plugins = Plugin_List(
g_pool, session_settings.get("loaded_plugins", default_plugins)
)
# Manually add g_pool.capture to the plugin list
g_pool.plugins._plugins.append(g_pool.capture)
g_pool.plugins._plugins.sort(key=lambda p: p.order)
g_pool.capture.init_ui()
general_settings.insert(
-1,
ui.Text_Input(
"rel_time_trim_section",
getter=g_pool.seek_control.get_rel_time_trim_range_string,
setter=g_pool.seek_control.set_rel_time_trim_range_string,
label="Relative time range to export",
),
)
general_settings.insert(
-1,
ui.Text_Input(
"frame_idx_trim_section",
getter=g_pool.seek_control.get_frame_index_trim_range_string,
setter=g_pool.seek_control.set_frame_index_trim_range_string,
label="Frame index range to export",
),
)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
toggle_general_settings(True)
g_pool.gui.configuration = session_settings.get("ui_config", {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n["subject"]
if subject == "start_plugin":
g_pool.plugins.add(
g_pool.plugin_by_name[n["name"]], args=n.get("args", {})
)
elif subject.startswith("meta.should_doc"):
ipc_pub.notify(
{"subject": "meta.doc", "actor": g_pool.app, "doc": player.__doc__}
)
for p in g_pool.plugins:
if (
p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify
):
ipc_pub.notify(
{
"subject": "meta.doc",
"actor": p.class_name,
"doc": p.on_notify.__doc__,
}
)
while (
not glfw.glfwWindowShouldClose(main_window) and not process_was_interrupted
):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
events = {}
# report time between now and the last loop interation
events["dt"] = get_dt()
# pupil and gaze positions are added by their respective producer plugins
events["pupil"] = []
events["gaze"] = []
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
glfw.glfwPollEvents()
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(
main_window, user_input.clipboard.encode()
)
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
# present frames at appropriate speed
g_pool.seek_control.wait(events["frame"].timestamp)
glfw.glfwSwapBuffers(main_window)
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["min_data_confidence"] = g_pool.min_data_confidence
session_settings[
"min_calibration_confidence"
] = g_pool.min_calibration_confidence
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
session_settings["version"] = str(g_pool.version)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except Exception:
import traceback
trace = traceback.format_exc()
logger.error("Process Player crashed with trace:\n{}".format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({"subject": "player_process.stopped"})
sleep(1.0)
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version,
eye_id, cap_src):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
is_alive = Is_Alive_Manager(is_alive_flag)
with is_alive:
#logging setup: We stream all log records to the world process.
import logging
import zmq
ctx = zmq.Context()
pub = ctx.socket(zmq.PUB)
pub.connect('tcp://127.0.0.1:502020')
class ZMQ_handler(logging.Handler):
def emit(self, record):
pub.send_pyobj(record)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(ZMQ_handler())
# create logger for the context of this function
logger = logging.getLogger(__name__)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen, make_coord_system_pixel_based, make_coord_system_norm_based, make_coord_system_eye_camera_based
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from av_writer import JPEG_Writer, AV_Writer
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D
}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600, 31 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# Callback functions
def on_resize(window, w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_button(window, button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(
pos,
(frame.width, frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size + 40,
g_pool.u_r.handle_size + 40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, (frame.width, frame.height))
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_eye%s' % eye_id))
if session_settings.get("version",
VersionFormat('0.0')) < g_pool.version:
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size': (640, 480), 'frame_rate': 60}
previous_settings = session_settings.get('capture_settings', None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
# pipe_to_world.send('eye%s process ready'%eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip', False)
g_pool.display_mode = session_settings.get('display_mode',
'camera_image')
g_pool.display_mode_info_text = {
'camera_image':
"Raw eye camera image. This uses the least amount of CPU power",
'roi':
"Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm':
"Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."
}
g_pool.u_r = UIRoi(frame.img.shape)
g_pool.u_r.set(session_settings.get('roi', g_pool.u_r.get()))
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1], new_size[0]))
cap.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get(
'pupil_detector_settings', None)
last_pupil_detector = pupil_detectors[session_settings.get(
'last_pupil_detector', Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,
pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s" % eye_id
width, height = session_settings.get('window_size',
(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfw.glfwSwapInterval(0)
sphere = Sphere(20)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",
pos=(-300, 0),
size=(0, 0),
header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=1.,
max=2.5,
label='Interface Size'))
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, frame.width, frame.height)))
general_settings.append(
ui.Switch('flip', g_pool, label='Flip image display'))
general_settings.append(
ui.Selector('display_mode',
g_pool,
setter=set_display_mode_info,
selection=['camera_image', 'roi', 'algorithm'],
labels=['Camera Image', 'ROI', 'Algorithm'],
label="Mode"))
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector(
'pupil_detector',
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Detector_2D, Detector_3D],
labels=['C++ 2d detector', 'C++ 3d detector'],
label="Detection method")
general_settings.append(detector_selector)
# let detector add its GUI
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config', {})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1 / 60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if pipe_to_world.poll():
cmd = pipe_to_world.recv()
if cmd == 'Exit':
break
elif cmd == "Ping":
pipe_to_world.send("Pong")
command = None
else:
command, payload = cmd
if command == 'Set_Detection_Mapping_Mode':
if payload == '3d':
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
set_detector(Detector_2D)
detector_selector.read_only = False
else:
command = None
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
cap.seek_to_frame(0)
frame = cap.get_frame()
#update performace graphs
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1. / dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if 'Rec_Start' == command:
record_path, raw_mode = payload
logger.info("Will save eye video to: %s" % record_path)
timestamps_path = os.path.join(record_path,
"eye%s_timestamps.npy" % eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = JPEG_Writer(video_path, cap.frame_rate)
else:
video_path = os.path.join(record_path,
"eye%s.mp4" % eye_id)
writer = AV_Writer(video_path, cap.frame_rate)
timestamps = []
elif 'Rec_Stop' == command:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image', 'roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based(
(frame.height, frame.width, 3), g_pool.flip)
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly(
(int(eye_ball['center'][0]),
int(eye_ball['center'][1])),
(int(eye_ball['axes'][0] / 2),
int(eye_ball['axes'][1] / 2)),
int(eye_ball['angle']), 0, 360, 8)
except ValueError as e:
pass
else:
draw_polyline(
pts, 2,
RGBA(0., .9, .1, result['model_confidence']))
if result['confidence'] > 0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly(
(int(result['ellipse']['center'][0]),
int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0] / 2),
int(result['ellipse']['axes'][1] / 2)),
int(result['ellipse']['angle']), 0, 360, 15)
confidence = result[
'confidence'] * 0.7 #scale it a little
draw_polyline(pts, 1, RGBA(1., 0, 0, confidence))
draw_points([result['ellipse']['center']],
size=20,
color=RGBA(1., 0., 0., confidence),
sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == 'roi':
g_pool.u_r.draw_points(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize(
) #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = g_pool.version
session_settings[
'last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings[
'pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
logger.info("Process Shutting down.")
def session(rec_dir):
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos,
(frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(
glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x * hdpi_factor, y * hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * y_scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x]
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '%s'" % new_rec_dir)
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'%s' is not a valid pupil recording" %
new_rec_dir)
tick = delta_t()
def get_dt():
return next(tick)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if f[-3:] in ('mp4', 'mkv', 'avi')
][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
#parse info.csv file
meta_info_path = os.path.join(rec_dir, "info.csv")
with open(meta_info_path) as info:
meta_info = dict(
((line.strip().split('\t')) for line in info.readlines()))
rec_version = read_rec_version(meta_info)
if rec_version >= VersionFormat('0.5'):
pass
elif rec_version >= VersionFormat('0.4'):
update_recording_0v4_to_current(rec_dir)
elif rec_version >= VersionFormat('0.3'):
update_recording_0v3_to_current(rec_dir)
timestamps_path = os.path.join(rec_dir, "timestamps.npy")
else:
logger.Error("This recording is to old. Sorry.")
return
timestamps = np.load(timestamps_path)
# Initialize capture
cap = File_Capture(video_path, timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version",
VersionFormat('0.0')) < get_version(version_file):
logger.info(
"Session setting are from older version of this app. I will not use those."
)
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', (0, 0))
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = get_version(version_file)
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
] #populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g_pool.new_seek = True
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Growing_Menu("Settings",
pos=(-350, 20),
size=(300, 400))
g_pool.main_menu.append(
ui.Button("Close Pupil Player",
lambda: glfwSetWindowShouldClose(main_window, True)))
g_pool.main_menu.append(
ui.Slider('scale',
g_pool.gui,
setter=set_scale,
step=.05,
min=0.75,
max=2.5,
label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: %s' %
g_pool.version))
g_pool.main_menu.append(ui.Info_Text('Recording Version: %s' %
rec_version))
g_pool.main_menu.append(
ui.Selector('Open plugin',
selection=user_launchable_plugins,
labels=[
p.__name__.replace('_', ' ')
for p in user_launchable_plugins
],
setter=open_plugin,
getter=lambda: "Select to load"))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(
ui.Button(
'Reset window size', lambda: glfwSetWindowSize(
main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label='Play',
hotkey=GLFW_KEY_SPACE)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT)
g_pool.backward_button = ui.Thumb('backward',
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT)
g_pool.quickbar.extend(
[g_pool.play_button, g_pool.forward_button, g_pool.backward_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
#we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Scan_Path', {}),
('Vis_Polyline', {}), ('Vis_Circle', {}),
('Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool, plugin_by_name,
system_plugins + previous_plugins)
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks':
g_pool.trim_marks = p
break
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
#trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
#set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = cap.get_timestamp() - .03
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
#grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g_pool.play = False
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
#report time between now and the last loop interation
events['dt'] = get_dt()
#new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(
g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(
g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
#update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts != t:
dt, ts = t - ts, t
fps_graph.add(1. / dt)
g_pool.play_button.status_text = str(frame.index)
#always update the CPU graph
cpu_graph.update()
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
#check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
#present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.close()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
def ui_elements(self):
ui_elements = []
if self.uvc_capture is None:
ui_elements.append(ui.Info_Text("Local USB: camera disconnected!"))
return ui_elements
ui_elements.append(ui.Info_Text(f"Camera: {self.name} @ Local USB"))
# lets define some helper functions:
def gui_load_defaults():
for c in self.uvc_capture.controls:
try:
c.value = c.def_val
except Exception:
pass
def gui_update_from_device():
for c in self.uvc_capture.controls:
c.refresh()
def set_frame_size(new_size):
self.frame_size = new_size
def set_frame_rate(new_rate):
self.frame_rate = new_rate
self.update_menu()
sensor_control = ui.Growing_Menu(label="Sensor Settings")
sensor_control.append(
ui.Info_Text("Do not change these during calibration or recording!")
)
sensor_control.collapsed = False
image_processing = ui.Growing_Menu(label="Image Post Processing")
image_processing.collapsed = True
sensor_control.append(
ui.Selector(
"frame_size",
self,
setter=set_frame_size,
selection=self.uvc_capture.frame_sizes,
label="Resolution",
)
)
def frame_rate_getter():
return (
self.uvc_capture.frame_rates,
[str(fr) for fr in self.uvc_capture.frame_rates],
)
# TODO: potential race condition through selection_getter. Should ensure that
# current selection will always be present in the list returned by the
# selection_getter. Highly unlikely though as this needs to happen between
# having clicked the Selector and the next redraw.
# See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b
sensor_control.append(
ui.Selector(
"frame_rate",
self,
selection_getter=frame_rate_getter,
setter=set_frame_rate,
label="Frame rate",
)
)
if (
"Pupil Cam2" in self.uvc_capture.name
or "Pupil Cam3" in self.uvc_capture.name
):
special_settings = {200: 28, 180: 31}
def set_exposure_mode(exposure_mode):
self.exposure_mode = exposure_mode
if self.exposure_mode == "auto":
self.preferred_exposure_time = Exposure_Time(
max_ET=special_settings.get(self.frame_rate, 32),
frame_rate=self.frame_rate,
mode=self.exposure_mode,
)
else:
self.preferred_exposure_time = None
logger.info(
"Exposure mode for camera {0} is now set to {1} mode".format(
self.uvc_capture.name, exposure_mode
)
)
self.update_menu()
sensor_control.append(
ui.Selector(
"exposure_mode",
self,
setter=set_exposure_mode,
selection=["manual", "auto"],
labels=["manual mode", "auto mode"],
label="Exposure Mode",
)
)
sensor_control.append(
ui.Slider(
"exposure_time",
self,
label="Absolute Exposure Time",
min=1,
max=special_settings.get(self.frame_rate, 32),
step=1,
)
)
if self.exposure_mode == "auto":
sensor_control[-1].read_only = True
if "Pupil Cam" in self.uvc_capture.name:
blacklist = [
"Auto Focus",
"Absolute Focus",
"Absolute Iris ",
"Scanning Mode ",
"Zoom absolute control",
"Pan control",
"Tilt control",
"Roll absolute control",
"Privacy Shutter control",
]
else:
blacklist = []
if (
"Pupil Cam2" in self.uvc_capture.name
or "Pupil Cam3" in self.uvc_capture.name
):
blacklist += [
"Auto Exposure Mode",
"Auto Exposure Priority",
"Absolute Exposure Time",
]
for control in self.uvc_capture.controls:
c = None
ctl_name = control.display_name
if ctl_name in blacklist:
continue
# now we add controls
if control.d_type == bool:
c = ui.Switch(
"value",
control,
label=ctl_name,
on_val=control.max_val,
off_val=control.min_val,
)
elif control.d_type == int:
c = ui.Slider(
"value",
control,
label=ctl_name,
min=control.min_val,
max=control.max_val,
step=control.step,
)
elif type(control.d_type) == dict:
selection = [value for name, value in control.d_type.items()]
labels = [name for name, value in control.d_type.items()]
c = ui.Selector(
"value", control, label=ctl_name, selection=selection, labels=labels
)
else:
pass
# if control['disabled']:
# c.read_only = True
# if ctl_name == 'Exposure, Auto Priority':
# # the controll should always be off. we set it to 0 on init (see above)
# c.read_only = True
if c is not None:
if control.unit == "processing_unit":
image_processing.append(c)
else:
sensor_control.append(c)
ui_elements.append(sensor_control)
if image_processing.elements:
ui_elements.append(image_processing)
ui_elements.append(ui.Button("refresh", gui_update_from_device))
if "Pupil Cam2" in self.uvc_capture.name:
def set_check_stripes(enable_stripe_checks):
self.enable_stripe_checks = enable_stripe_checks
if self.enable_stripe_checks:
self.stripe_detector = Check_Frame_Stripes()
logger.info(
"Check Stripes for camera {} is now on".format(
self.uvc_capture.name
)
)
else:
self.stripe_detector = None
logger.info(
"Check Stripes for camera {} is now off".format(
self.uvc_capture.name
)
)
ui_elements.append(
ui.Switch(
"enable_stripe_checks",
self,
setter=set_check_stripes,
label="Check Stripes",
)
)
return ui_elements
def init_ui(self):
super().init_ui()
self.menu.label = "Offline Calibration"
self.glfont = fontstash.Context()
self.glfont.add_font('opensans', ui.get_opensans_font_path())
self.glfont.set_color_float((1., 1., 1., 1.))
self.glfont.set_align_string(v_align='right', h_align='top')
def use_as_natural_features():
self.manual_ref_positions.extend(self.circle_marker_positions)
self.manual_ref_positions.sort(key=lambda mr: mr['index'])
def jump_next_natural_feature():
self.manual_ref_positions.sort(key=lambda mr: mr['index'])
current = self.g_pool.capture.get_frame_index()
for nf in self.manual_ref_positions:
if nf['index'] > current:
self.notify_all({
'subject': 'seek_control.should_seek',
'index': nf['index']
})
return
logger.error('No further natural feature available')
def clear_natural_features():
self.manual_ref_positions = []
self.menu.append(
ui.Info_Text(
'"Detection" searches for circle markers in the world video.'))
# self.menu.append(ui.Button('Redetect', self.start_marker_detection))
slider = ui.Slider('detection_progress',
self,
label='Detection Progress',
setter=lambda _: _)
slider.display_format = '%3.0f%%'
self.menu.append(slider)
toggle_label = 'Cancel circle marker detection' if self.process_pipe else 'Start circle marker detection'
self.toggle_detection_button = ui.Button(toggle_label,
self.toggle_marker_detection)
self.menu.append(self.toggle_detection_button)
self.menu.append(ui.Separator())
self.menu.append(
ui.Button('Use calibration markers as natural features',
use_as_natural_features))
self.menu.append(
ui.Button('Jump to next natural feature',
jump_next_natural_feature))
self.menu.append(
ui.Switch('manual_ref_edit_mode',
self,
label="Natural feature edit mode"))
self.menu.append(
ui.Button('Clear natural features', clear_natural_features))
self.menu.append(
ui.Info_Text(
'Calibration only considers pupil data that has an equal or higher confidence than the minimum calibration confidence.'
))
self.menu.append(
ui.Slider('min_calibration_confidence',
self.g_pool,
step=.01,
min=0.0,
max=1.0,
label='Minimum calibration confidence'))
self.menu.append(ui.Button('Add section', self.append_section))
# set to minimum height
self.timeline = ui.Timeline('Calibration Sections', self.draw_sections,
self.draw_labels, 1)
self.g_pool.user_timelines.append(self.timeline)
for sec in self.sections:
self.append_section_menu(sec)
self.on_window_resize(
glfw.glfwGetCurrentContext(),
*glfw.glfwGetWindowSize(glfw.glfwGetCurrentContext()))
